@klhapp/skillmux 1.0.1 → 1.2.0

package/src/adapters.ts

@@ -1,8 +1,9 @@
-import { Database } from "bun:sqlite";
+import { createHash } from "node:crypto";
 import { join } from "node:path";
-import { applyCalibrationRun, getCalibrationRun, listCalibrationRuns, loadDecisionCasesFromFile, openCalibrateDb, runCalibration, type CalibrationResult } from "./calibrate";
+import { applyCalibrationRun, getCalibrationRun, insertCalibrationRun, listCalibrationRuns, loadDecisionCasesFromFile, openCalibrateDb, runCalibration, summarizeDatasetProvenance, type CalibrationResult } from "./calibrate";
 import { createClients } from "./clients";
-import { DEFAULT_CONFIG_PATH, expandHome, loadConfig } from "./config";
+import { DEFAULT_CONFIG_PATH, embeddingFingerprint, expandHome, loadConfig, rerankerFingerprint } from "./config";
+import { openIndex } from "./db";
 import { CliError } from "./output";
 import {
   computeHash,
@@ -18,7 +19,7 @@ import {
   type SetConfigResult,
 } from "./config-service";
 import type { ResolvedTarget } from "./context";
-import { resolveSkill } from "./router-core";
+import { configure, retrieveAndRerank } from "./router-core";
 import type { Config } from "./types";
 
 export interface Capabilities {
@@ -43,7 +44,13 @@ export interface TargetAdapter {
   configDiff(): Promise<{ diff: Record<string, { prior: unknown; resulting: unknown }> }>;
   configSet(key: string, rawValStr: string, opts?: { dryRun?: boolean }): Promise<SetConfigResult>;
   configStatus(): Promise<ConfigStatusResponse>;
-  calibrateRun(opts?: { datasetPath?: string }): Promise<{ run_id?: string; result?: CalibrationResult }>;
+  calibrateRun(opts?: {
+    datasetPath?: string;
+    minAutoMatchPrecision?: number;
+    minRetrievalRecallAtK?: number;
+    minDeliveredShortlistRecallAtK?: number;
+    minAutoMatchCount?: number;
+  }): Promise<{ run_id?: string; result?: CalibrationResult }>;
   calibrateList(): Promise<any[]>;
   calibrateShow(runId: string): Promise<any>;
   calibrateApply(runId: string): Promise<any>;
@@ -123,26 +130,89 @@ export class LocalAdapter implements TargetAdapter {
     return getLocalConfigStatus(this.configPath);
   }
 
-  async calibrateRun(opts?: { datasetPath?: string }): Promise<{ run_id?: string; result?: CalibrationResult }> {
+  async calibrateRun(opts?: {
+    datasetPath?: string;
+    minAutoMatchPrecision?: number;
+    minRetrievalRecallAtK?: number;
+    minDeliveredShortlistRecallAtK?: number;
+    minAutoMatchCount?: number;
+  }): Promise<{ run_id?: string; result?: CalibrationResult }> {
     const config = await loadConfig(this.configPath);
     const datasetFile = opts?.datasetPath ?? join(expandHome(config.state_dir), "queries.json");
-    const cases = loadDecisionCasesFromFile(datasetFile);
+    const indexDb = openIndex(expandHome(config.state_dir));
+    let indexedSkills: Array<{ skill_id: string; content_sha256: string }>;
+    try {
+      indexedSkills = indexDb
+        .query("SELECT skill_id, content_sha256 FROM skills ORDER BY skill_id")
+        .all() as Array<{ skill_id: string; content_sha256: string }>;
+    } finally {
+      indexDb.close();
+    }
+    const cases = loadDecisionCasesFromFile(
+      datasetFile,
+      indexedSkills.map((skill) => skill.skill_id),
+    );
     const clients = createClients(config);
+    configure({ config, clients });
     const result = await runCalibration({
       cases,
-      getCandidates: async (query: string) => {
-        const res = await resolveSkill({ query, forceLexical: false });
-        if (res.outcome === "matched") {
-          return [{ skill_id: res.skill_id, text: `${res.title} ${res.body}` }];
+      getRankedCandidates: async (query: string) => {
+        const result = await retrieveAndRerank({ query, forceLexical: false });
+        if (result.retrieval !== "reranked") {
+          throw new Error(
+            "Calibration requires successful hybrid retrieval and reranking for every query.",
+          );
         }
-        if (res.outcome === "ambiguous") {
-          return res.candidates.map((c) => ({ skill_id: c.skill_id, text: `${c.title} ${c.description}` }));
-        }
-        return [];
+        return result.candidates.map((candidate) => ({
+          skill_id: candidate.skill_id,
+          score: candidate.score ?? 0,
+        }));
       },
       reranker: clients.rerank,
+      candidateLimit: config.thresholds.candidate_limit,
+      minAutoMatchPrecision: opts?.minAutoMatchPrecision,
+      minRetrievalRecallAtK: opts?.minRetrievalRecallAtK,
+      minDeliveredShortlistRecallAtK: opts?.minDeliveredShortlistRecallAtK,
+      minAutoMatchCount: opts?.minAutoMatchCount,
     });
-    return { result };
+    const fingerprint = rerankerFingerprint(config);
+    if (!fingerprint) {
+      throw new Error("A configured remote reranker is required to record calibration.");
+    }
+    const datasetText = await Bun.file(datasetFile).text();
+    const corpusFingerprint =
+      "vault:" +
+      createHash("sha256").update(JSON.stringify(indexedSkills)).digest("hex");
+    const runId = `run_${crypto.randomUUID()}`;
+    const db = openCalibrateDb(expandHome(config.state_dir));
+    try {
+      insertCalibrationRun(db, {
+        run_id: runId,
+        created_at: new Date().toISOString(),
+        status: result.status,
+        reranker_fingerprint: fingerprint,
+        embedding_fingerprint: embeddingFingerprint(config),
+        corpus_fingerprint: corpusFingerprint,
+        dataset_hash: createHash("sha256").update(datasetText).digest("hex"),
+        dataset_provenance: summarizeDatasetProvenance(cases),
+        candidate_limit: config.thresholds.candidate_limit,
+        min_auto_match_precision: opts?.minAutoMatchPrecision ?? 0.99,
+        min_auto_match_count: opts?.minAutoMatchCount ?? 30,
+        min_delivered_shortlist_recall_at_k:
+          opts?.minDeliveredShortlistRecallAtK ??
+          opts?.minRetrievalRecallAtK ??
+          0.95,
+        min_shortlist_recall_at_5: opts?.minRetrievalRecallAtK ?? 0.95,
+        failed_reason: result.failed_reason,
+        selected_thresholds: result.selected_thresholds,
+        tune_metrics: result.tune_metrics,
+        test_metrics: result.test_metrics,
+        observations: result.observations,
+      });
+    } finally {
+      db.close();
+    }
+    return { run_id: runId, result };
   }
 
   async calibrateList(): Promise<any[]> {
@@ -173,7 +243,9 @@ export class LocalAdapter implements TargetAdapter {
     try {
       const run = getCalibrationRun(db, runId);
       if (!run) throw new Error(`Calibration run "${runId}" not found`);
-      await applyCalibrationRun(db, runId, expandHome(this.configPath), {});
+      await applyCalibrationRun(db, runId, expandHome(this.configPath), {
+        currentRerankerFingerprint: rerankerFingerprint(config),
+      });
       return { ok: true, run_id: runId };
     } finally {
       db.close();
@@ -345,36 +417,37 @@ export class RemoteAdapter implements TargetAdapter {
     return data.runtime;
   }
 
-  async calibrateRun(opts?: { datasetPath?: string }): Promise<{ run_id?: string; result?: CalibrationResult }> {
-    const { status, data } = await this.fetchJson("/admin/v1/calibrations", {
-      method: "POST",
-      headers: { "Content-Type": "application/json" },
-      body: JSON.stringify({ dataset_path: opts?.datasetPath }),
-    });
-    if (status !== 202) {
-      throw new Error(`Remote calibration start failed (${status}): ${data?.message || data}`);
-    }
-    return data;
+  async calibrateRun(opts?: {
+    datasetPath?: string;
+    minAutoMatchPrecision?: number;
+    minRetrievalRecallAtK?: number;
+    minDeliveredShortlistRecallAtK?: number;
+    minAutoMatchCount?: number;
+  }): Promise<{ run_id?: string; result?: CalibrationResult }> {
+    void opts;
+    throw this.remoteCalibrationNotImplemented();
   }
 
   async calibrateList(): Promise<any[]> {
-    const { status, data } = await this.fetchJson("/admin/v1/calibrations");
-    if (status !== 200) throw new Error(`Remote calibration list failed (${status}): ${data?.message || data}`);
-    return data;
+    throw this.remoteCalibrationNotImplemented();
   }
 
   async calibrateShow(runId: string): Promise<any> {
-    const { status, data } = await this.fetchJson(`/admin/v1/calibrations/${runId}`);
-    if (status !== 200) throw new Error(`Remote calibration show failed (${status}): ${data?.message || data}`);
-    return data;
+    void runId;
+    throw this.remoteCalibrationNotImplemented();
   }
 
   async calibrateApply(runId: string): Promise<any> {
-    const { status, data } = await this.fetchJson(`/admin/v1/calibrations/${runId}/apply`, {
-      method: "POST",
-    });
-    if (status !== 200) throw new Error(`Remote calibration apply failed (${status}): ${data?.message || data}`);
-    return data;
+    void runId;
+    throw this.remoteCalibrationNotImplemented();
+  }
+
+  private remoteCalibrationNotImplemented(): CliError {
+    return new CliError(
+      `Remote calibration is not implemented for target ${this.serverUrl}; ` +
+        "run `skillmux calibrate` against a local target.",
+      2,
+    );
   }
 }