@klhapp/skillmux 1.5.1 → 1.6.0

package/src/adapters.ts

@@ -1,6 +1,23 @@
 import { createHash } from "node:crypto";
 import { join } from "node:path";
-import { applyCalibrationRun, computeCorpusFingerprint, getCalibrationRun, insertCalibrationRun, listCalibrationRuns, loadDecisionCasesFromFile, openCalibrateDb, runCalibration, summarizeDatasetProvenance, type CalibrationResult } from "./calibrate";
+import {
+  applyCalibrationRun,
+  assertCalibrationFeasibility,
+  computeCorpusFingerprint,
+  createInitialCalibrationRun,
+  finalizeCalibrationRun,
+  getCalibrationObservations,
+  getCalibrationRun,
+  insertCalibrationRun,
+  listCalibrationRuns,
+  loadDecisionCasesFromFile,
+  openCalibrateDb,
+  runCalibration,
+  saveCalibrationObservation,
+  summarizeDatasetProvenance,
+  type CalibrationResult,
+  type QueryObservation,
+} from "./calibrate";
 import { createClients } from "./clients";
 import { embeddingFingerprint, expandHome, loadConfig, rerankerFingerprint, resolveConfigPath } from "./config";
 import { openIndex } from "./db";
@@ -19,8 +36,33 @@ import {
   type SetConfigResult,
 } from "./config-service";
 import type { ResolvedTarget } from "./context";
-import { configure, retrieveAndRerank } from "./router-core";
-import type { Config } from "./types";
+import {
+  classifyInferenceError,
+  configure,
+  retrieveAndRerankSnapshot,
+  syncVaultIfNeeded,
+  type StageTiming,
+} from "./router-core";
+import type { Clients, Config } from "./types";
+
+/**
+ * Maximum retry attempts for transient reranker availability errors during calibration.
+ *
+ * Single-capacity remote reranker endpoints can suffer isolated blips (e.g. transient
+ * connection reset or momentary 503) during long multi-case calibration runs.
+ * A small bounded budget of 2 retries (3 attempts total) allows calibration to ride out
+ * transient blips without stalling on hard failures.
+ */
+export const CALIBRATION_RERANK_MAX_RETRIES = 2;
+
+/**
+ * Base backoff delay (ms) between reranker retries during calibration.
+ *
+ * A conservative nonzero delay gives recovering remote endpoints time to settle.
+ * Because reranker admission is strictly FIFO serialized, waiting workers remain
+ * queued rather than creating synchronized retry storms.
+ */
+export const CALIBRATION_RERANK_RETRY_BACKOFF_MS = 250;
 
 export interface Capabilities {
   config_read: boolean;
@@ -34,6 +76,44 @@ export interface Capabilities {
 export interface TargetAdapterOptions {
   configPath?: string;
   allowInsecure?: boolean;
+  clients?: Clients;
+}
+
+/**
+ * Aggregate timing data collected during a single calibrateRun invocation.
+ *
+ * All durations are in milliseconds and are non-negative.
+ *
+ * Cumulative fields (cumulative_embedding_ms, cumulative_lexical_ms,
+ * cumulative_vector_ms, cumulative_reranker_ms, cumulative_checkpoint_ms)
+ * represent total worker time summed across all concurrent query retrievals
+ * or checkpoint writes. Because queries run concurrently, the sum of these
+ * cumulative fields may exceed wall_ms — they measure how much worker time
+ * each stage consumed, not how much wall-clock time it contributed.
+ */
+export interface CalibrationTimingSummary {
+  /** Total number of dataset cases. */
+  cases_total: number;
+  /** Cases actually retrieved in this invocation (not reused from a prior run). */
+  cases_executed: number;
+  /** Cases loaded from a prior interrupted run (resume observations). */
+  cases_reused: number;
+  /** Wall-clock duration of the full calibrateRun operation (ms). */
+  wall_ms: number;
+  /** Duration of the one-time vault synchronization before retrieval (ms). */
+  vault_sync_ms: number;
+  /** Cumulative worker time spent in embedding across all queries (ms). */
+  cumulative_embedding_ms: number;
+  /** Cumulative worker time spent in lexical search across all queries (ms). */
+  cumulative_lexical_ms: number;
+  /** Cumulative worker time spent in vector search across all queries (ms). */
+  cumulative_vector_ms: number;
+  /** Cumulative worker time spent in reranking across all queries (ms). */
+  cumulative_reranker_ms: number;
+  /** Cumulative worker time spent writing observation checkpoints (ms). */
+  cumulative_checkpoint_ms: number;
+  /** Duration of threshold selection and test-split certification (ms). */
+  policy_evaluation_ms: number;
 }
 
 export interface TargetAdapter {
@@ -50,6 +130,16 @@ export interface TargetAdapter {
     minRetrievalRecallAtK?: number;
     minDeliveredShortlistRecallAtK?: number;
     minAutoMatchCount?: number;
+    tuneAutoMatchPrecisionBuffer?: number;
+    tuneAutoMatchCountBuffer?: number;
+    tuneDeliveredShortlistRecallBuffer?: number;
+    concurrency?: number;
+    resumeRunId?: string;
+    onProgress?: (completed: number, total: number) => void;
+    /** Opt-in timing collection. When true, onTimingSummary is called after a non-throwing result. */
+    timing?: boolean;
+    /** Called after calibration completes or fails-gates (not called on throws). */
+    onTimingSummary?: (summary: CalibrationTimingSummary) => void;
   }): Promise<{ run_id?: string; result?: CalibrationResult }>;
   calibrateList(): Promise<any[]>;
   calibrateShow(runId: string): Promise<any>;
@@ -68,9 +158,11 @@ export function isLoopbackHost(hostname: string): boolean {
 
 export class LocalAdapter implements TargetAdapter {
   private configPath: string;
+  private clients?: Clients;
 
   constructor(opts?: TargetAdapterOptions) {
     this.configPath = resolveConfigPath(opts?.configPath);
+    this.clients = opts?.clients;
   }
 
   async getCapabilities(): Promise<Capabilities> {
@@ -119,6 +211,7 @@ export class LocalAdapter implements TargetAdapter {
     if (caps.persistence === "externally_managed") {
       throw new CliError("Configuration is externally managed and cannot be modified", 4);
     }
+    validateDottedKey(key);
     return setDottedKey(key, rawValStr, {
       configPath: this.configPath,
       dryRun: opts?.dryRun,
@@ -136,10 +229,74 @@ export class LocalAdapter implements TargetAdapter {
     minRetrievalRecallAtK?: number;
     minDeliveredShortlistRecallAtK?: number;
     minAutoMatchCount?: number;
+    tuneAutoMatchPrecisionBuffer?: number;
+    tuneAutoMatchCountBuffer?: number;
+    tuneDeliveredShortlistRecallBuffer?: number;
+    concurrency?: number;
+    resumeRunId?: string;
+    onProgress?: (completed: number, total: number) => void;
+    timing?: boolean;
+    onTimingSummary?: (summary: CalibrationTimingSummary) => void;
   }): Promise<{ run_id?: string; result?: CalibrationResult }> {
+    const collectTiming = opts?.timing === true;
+    const wallStart = collectTiming ? performance.now() : 0;
+
     const config = await loadConfig(this.configPath);
+    const baseClients = this.clients ?? createClients(config);
+
+    // Bounded admission at the calibration reranker boundary: serialize rerank calls
+    // so concurrent case retrieval (embedding, lexical, vector) does not overwhelm
+    // a single-capacity remote reranker endpoint.
+    //
+    // If an admitted request encounters a transient reranker_unavailable error,
+    // retry with conservative backoff up to CALIBRATION_RERANK_MAX_RETRIES.
+    // Non-availability errors (protocol, timeouts) and permanent failures fail closed.
+    let rerankQueue = Promise.resolve() as Promise<any>;
+    const serializedRerank: typeof baseClients.rerank = baseClients.rerank
+      ? (query, docs) => {
+          const run = async () => {
+            let attempt = 0;
+            while (true) {
+              try {
+                return await baseClients.rerank!(query, docs);
+              } catch (err) {
+                attempt++;
+                const degradationReason = classifyInferenceError("reranker", err);
+                if (
+                  degradationReason === "reranker_unavailable" &&
+                  attempt <= CALIBRATION_RERANK_MAX_RETRIES
+                ) {
+                  await Bun.sleep(CALIBRATION_RERANK_RETRY_BACKOFF_MS * attempt);
+                  continue;
+                }
+                throw err;
+              }
+            }
+          };
+          const next = rerankQueue.then(run, run);
+          rerankQueue = next.catch(() => {});
+          return next;
+        }
+      : undefined;
+
+    const clients: Clients = {
+      ...baseClients,
+      rerank: serializedRerank,
+    };
+    configure({ config, clients });
+
+    // Measure vault synchronization
+    let vault_sync_ms = 0;
+    if (collectTiming) {
+      const t0 = performance.now();
+      await syncVaultIfNeeded();
+      vault_sync_ms = Math.max(0, performance.now() - t0);
+    } else {
+      await syncVaultIfNeeded();
+    }
+
     const candidateLimit =
-      config.output?.ambiguous_candidate_limit ?? config.thresholds.candidate_limit ?? 5;
+      config.output?.ambiguous_candidate_limit ?? config.thresholds?.candidate_limit ?? 5;
     const datasetFile = opts?.datasetPath ?? join(expandHome(config.state_dir), "queries.json");
     const indexDb = openIndex(expandHome(config.state_dir));
     let indexedSkills: Array<{ skill_id: string; content_sha256: string }>;
@@ -156,71 +313,266 @@ export class LocalAdapter implements TargetAdapter {
       datasetFile,
       indexedSkills.map((skill) => skill.skill_id),
     );
-    const clients = createClients(config);
-    configure({ config, clients });
-    const result = await runCalibration({
-      cases,
-      getRankedCandidates: async (query: string) => {
-        const result = await retrieveAndRerank({ query, forceLexical: false });
-        if (result.retrieval !== "reranked") {
-          throw new Error(
-            "Calibration requires successful hybrid retrieval and reranking for every query.",
-          );
-        }
-        return result.candidates.map((candidate) => ({
-          skill_id: candidate.skill_id,
-          score: candidate.score ?? 0,
-        }));
-      },
-      reranker: clients.rerank,
-      candidateLimit,
-      minAutoMatchPrecision: opts?.minAutoMatchPrecision,
-      minRetrievalRecallAtK: opts?.minRetrievalRecallAtK,
-      minDeliveredShortlistRecallAtK: opts?.minDeliveredShortlistRecallAtK,
-      minAutoMatchCount: opts?.minAutoMatchCount,
-    });
+
     const fingerprint = rerankerFingerprint(config);
     if (!fingerprint) {
       throw new Error("A configured remote reranker is required to record calibration.");
     }
     const datasetText = await Bun.file(datasetFile).text();
-    const runId = `run_${crypto.randomUUID()}`;
+    const datasetHash = createHash("sha256").update(datasetText).digest("hex");
+    const embedFp = embeddingFingerprint(config);
+    const recallSettings = {
+      k_lexical: config.recall.k_lexical,
+      k_vector: config.recall.k_vector,
+      k_rerank: config.recall.k_rerank ?? Math.min(10, config.recall.k_lexical + config.recall.k_vector),
+    };
+    const minAutoMatchPrecision = opts?.minAutoMatchPrecision ?? 0.75;
+    const minAutoMatchCount = opts?.minAutoMatchCount ?? 15;
+    const minDeliveredShortlistRecallAtK =
+      opts?.minDeliveredShortlistRecallAtK ??
+      opts?.minRetrievalRecallAtK ??
+      0.95;
+    const minShortlistRecallAt5 = opts?.minRetrievalRecallAtK ?? 0.95;
+    const tuneAutoMatchPrecisionBuffer = opts?.tuneAutoMatchPrecisionBuffer ?? 0.03;
+    const tuneAutoMatchCountBuffer = opts?.tuneAutoMatchCountBuffer ?? 3;
+    const tuneDeliveredShortlistRecallBuffer = opts?.tuneDeliveredShortlistRecallBuffer ?? 0.02;
+    const concurrency = opts?.concurrency ?? 4;
+
+    assertCalibrationFeasibility(cases, {
+      minAutoMatchPrecision,
+      minAutoMatchCount,
+      minDeliveredShortlistRecallAtK,
+      tuneAutoMatchPrecisionBuffer,
+      tuneAutoMatchCountBuffer,
+      tuneDeliveredShortlistRecallBuffer,
+    });
+
     const db = openCalibrateDb(expandHome(config.state_dir));
+    let runId: string;
+    let initialObservations: Map<number, QueryObservation> | undefined = undefined;
+
+    // Cumulative timing accumulators (only used when collectTiming=true)
+    let cumulative_embedding_ms = 0;
+    let cumulative_lexical_ms = 0;
+    let cumulative_vector_ms = 0;
+    let cumulative_reranker_ms = 0;
+    let cumulative_checkpoint_ms = 0;
+
     try {
-      insertCalibrationRun(db, {
+      if (opts?.resumeRunId) {
+        runId = opts.resumeRunId;
+        const existingRun = getCalibrationRun(db, runId);
+        if (!existingRun) {
+          throw new Error(`Calibration run "${runId}" not found`);
+        }
+        if (existingRun.status !== "running") {
+          throw new Error(`Cannot resume completed calibration run "${runId}"`);
+        }
+        if (existingRun.dataset_hash !== datasetHash) {
+          throw new Error("Dataset hash mismatch: cannot resume run with a different dataset");
+        }
+        if (existingRun.corpus_fingerprint !== corpusFingerprint) {
+          throw new Error("Corpus fingerprint mismatch: cannot resume run with modified vault skills");
+        }
+        if (existingRun.embedding_fingerprint !== embedFp) {
+          throw new Error("Embedding fingerprint mismatch: cannot resume run with different embedding configuration");
+        }
+        if (existingRun.reranker_fingerprint !== fingerprint) {
+          throw new Error("Reranker fingerprint mismatch: cannot resume run with different reranker configuration");
+        }
+        if (existingRun.candidate_limit !== candidateLimit) {
+          throw new Error("Candidate limit mismatch: cannot resume run with different candidate limit");
+        }
+        if (
+          existingRun.recall_settings &&
+          JSON.stringify(existingRun.recall_settings) !== JSON.stringify(recallSettings)
+        ) {
+          throw new Error("Recall settings mismatch: cannot resume run with different recall parameters");
+        }
+        if (existingRun.min_auto_match_precision !== minAutoMatchPrecision) {
+          throw new Error("Certification gate mismatch: min_auto_match_precision differs from original run");
+        }
+        if ((existingRun.min_auto_match_count ?? 1) !== minAutoMatchCount) {
+          throw new Error("Certification gate mismatch: min_auto_match_count differs from original run");
+        }
+        if (
+          (existingRun.min_delivered_shortlist_recall_at_k ?? existingRun.min_shortlist_recall_at_5) !==
+          minDeliveredShortlistRecallAtK
+        ) {
+          throw new Error(
+            "Certification gate mismatch: min_delivered_shortlist_recall_at_k differs from original run",
+          );
+        }
+        if (existingRun.min_shortlist_recall_at_5 !== minShortlistRecallAt5) {
+          throw new Error("Certification gate mismatch: min_retrieval_recall_at_k differs from original run");
+        }
+        if (
+          existingRun.tune_auto_match_precision_buffer !== undefined &&
+          existingRun.tune_auto_match_precision_buffer !== tuneAutoMatchPrecisionBuffer
+        ) {
+          throw new Error(
+            "Certification gate mismatch: tune_auto_match_precision_buffer differs from original run",
+          );
+        }
+        if (
+          existingRun.tune_auto_match_count_buffer !== undefined &&
+          existingRun.tune_auto_match_count_buffer !== tuneAutoMatchCountBuffer
+        ) {
+          throw new Error(
+            "Certification gate mismatch: tune_auto_match_count_buffer differs from original run",
+          );
+        }
+        if (
+          existingRun.tune_delivered_shortlist_recall_buffer !== undefined &&
+          existingRun.tune_delivered_shortlist_recall_buffer !== tuneDeliveredShortlistRecallBuffer
+        ) {
+          throw new Error(
+            "Certification gate mismatch: tune_delivered_shortlist_recall_buffer differs from original run",
+          );
+        }
+
+        initialObservations = getCalibrationObservations(db, runId);
+        if (initialObservations.size === 0 && existingRun.observations && existingRun.observations.length > 0) {
+          existingRun.observations.forEach((obs, idx) => initialObservations!.set(idx, obs));
+        }
+      } else {
+        runId = `run_${crypto.randomUUID()}`;
+        createInitialCalibrationRun(db, {
+          run_id: runId,
+          created_at: new Date().toISOString(),
+          status: "running",
+          reranker_fingerprint: fingerprint,
+          embedding_fingerprint: embedFp,
+          corpus_fingerprint: corpusFingerprint,
+          dataset_hash: datasetHash,
+          dataset_provenance: summarizeDatasetProvenance(cases),
+          recall_settings: recallSettings,
+          candidate_limit: candidateLimit,
+          min_auto_match_precision: minAutoMatchPrecision,
+          min_auto_match_count: minAutoMatchCount,
+          min_delivered_shortlist_recall_at_k: minDeliveredShortlistRecallAtK,
+          min_shortlist_recall_at_5: minShortlistRecallAt5,
+          tune_auto_match_precision_buffer: tuneAutoMatchPrecisionBuffer,
+          tune_auto_match_count_buffer: tuneAutoMatchCountBuffer,
+          tune_delivered_shortlist_recall_buffer: tuneDeliveredShortlistRecallBuffer,
+        });
+      }
+
+      const casesReused = initialObservations?.size ?? 0;
+
+      const onProgress = opts?.onProgress ?? ((completed, total) => {
+        process.stderr.write(`Calibration observations: ${completed}/${total}\n`);
+      });
+
+      const retrievalTiming = collectTiming
+        ? {
+            onTiming: (timing: StageTiming) => {
+              cumulative_embedding_ms += timing.embedding_ms;
+              cumulative_lexical_ms += timing.lexical_ms;
+              cumulative_vector_ms += timing.vector_ms;
+              cumulative_reranker_ms += timing.reranker_ms;
+            },
+          }
+        : undefined;
+
+      const getRankedCandidates = async (query: string) => {
+        const res = await retrieveAndRerankSnapshot(
+          { query, forceLexical: false },
+          retrievalTiming,
+        );
+        if (res.retrieval !== "reranked") {
+          throw new Error(
+            "Calibration requires successful hybrid retrieval and reranking for every query.",
+          );
+        }
+        return res.candidates.map((candidate) => ({
+          skill_id: candidate.skill_id,
+          score: candidate.score ?? 0,
+        }));
+      };
+
+      let policyEvaluationStart = 0;
+
+      const result = await runCalibration({
+        cases,
+        getRankedCandidates,
+        reranker: clients.rerank,
+        candidateLimit,
+        minAutoMatchPrecision,
+        minRetrievalRecallAtK: minShortlistRecallAt5,
+        minDeliveredShortlistRecallAtK,
+        minAutoMatchCount,
+        tuneAutoMatchPrecisionBuffer,
+        tuneAutoMatchCountBuffer,
+        tuneDeliveredShortlistRecallBuffer,
+        concurrency,
+        initialObservations,
+        onProgress,
+        onObservation: collectTiming
+          ? (obs, caseIdx) => {
+              const t0 = performance.now();
+              saveCalibrationObservation(db, runId, caseIdx, obs);
+              const t1 = performance.now();
+              cumulative_checkpoint_ms += Math.max(0, t1 - t0);
+            }
+          : (obs, caseIdx) => {
+              saveCalibrationObservation(db, runId, caseIdx, obs);
+            },
+        onObservationsReady: collectTiming
+          ? () => {
+              policyEvaluationStart = performance.now();
+            }
+          : undefined,
+      });
+
+      // policy_evaluation_ms: threshold selection + certification duration
+      const policyEvalEnd = collectTiming ? performance.now() : 0;
+
+      finalizeCalibrationRun(db, {
         run_id: runId,
-        created_at: new Date().toISOString(),
         status: result.status,
-        reranker_fingerprint: fingerprint,
-        embedding_fingerprint: embeddingFingerprint(config),
-        corpus_fingerprint: corpusFingerprint,
-        dataset_hash: createHash("sha256").update(datasetText).digest("hex"),
-        dataset_provenance: summarizeDatasetProvenance(cases),
-        recall_settings: {
-          k_lexical: config.recall.k_lexical,
-          k_vector: config.recall.k_vector,
-          k_rerank: config.recall.k_rerank ?? Math.min(10, config.recall.k_lexical + config.recall.k_vector),
-        },
-        candidate_limit: candidateLimit,
-        min_auto_match_precision: opts?.minAutoMatchPrecision ?? 0.99,
-        min_auto_match_count: opts?.minAutoMatchCount ?? 30,
-        min_delivered_shortlist_recall_at_k:
-          opts?.minDeliveredShortlistRecallAtK ??
-          opts?.minRetrievalRecallAtK ??
-          0.95,
-        min_shortlist_recall_at_5: opts?.minRetrievalRecallAtK ?? 0.95,
         failed_reason: result.failed_reason,
         selected_thresholds: result.selected_thresholds,
         tune_metrics: result.tune_metrics,
         test_metrics: result.test_metrics,
+        tune_gate_slack: result.tune_gate_slack,
         observations: result.observations,
       });
+
+      // Emit timing summary after a non-throwing result (completed or failed-gates).
+      // Never called when calibrateRun throws.
+      if (collectTiming && opts?.onTimingSummary) {
+        const wall_ms = Math.max(0, performance.now() - wallStart);
+        // policy_evaluation_ms covers threshold selection + test-split certification,
+        // the internal work runCalibration does after all observations are collected.
+        const policy_evaluation_ms = Math.max(0, policyEvalEnd - policyEvaluationStart);
+
+        const cases_reused = casesReused;
+        const cases_total = cases.length;
+        const cases_executed = cases_total - cases_reused;
+
+        opts.onTimingSummary({
+          cases_total,
+          cases_executed,
+          cases_reused,
+          wall_ms,
+          vault_sync_ms,
+          cumulative_embedding_ms,
+          cumulative_lexical_ms,
+          cumulative_vector_ms,
+          cumulative_reranker_ms,
+          cumulative_checkpoint_ms,
+          policy_evaluation_ms,
+        });
+      }
+
+      return { run_id: runId, result };
     } finally {
       db.close();
     }
-    return { run_id: runId, result };
   }
 
+
   async calibrateList(): Promise<any[]> {
     const config = await loadConfig(this.configPath);
     const db = openCalibrateDb(expandHome(config.state_dir));
@@ -429,11 +781,20 @@ export class RemoteAdapter implements TargetAdapter {
     minRetrievalRecallAtK?: number;
     minDeliveredShortlistRecallAtK?: number;
     minAutoMatchCount?: number;
+    tuneAutoMatchPrecisionBuffer?: number;
+    tuneAutoMatchCountBuffer?: number;
+    tuneDeliveredShortlistRecallBuffer?: number;
+    concurrency?: number;
+    resumeRunId?: string;
+    onProgress?: (completed: number, total: number) => void;
+    timing?: boolean;
+    onTimingSummary?: (summary: CalibrationTimingSummary) => void;
   }): Promise<{ run_id?: string; result?: CalibrationResult }> {
     void opts;
     throw this.remoteCalibrationNotImplemented();
   }
 
+
   async calibrateList(): Promise<any[]> {
     throw this.remoteCalibrationNotImplemented();
   }