@klhapp/skillmux 1.5.1 → 1.6.0

package/src/calibrate.ts

@@ -322,7 +322,7 @@ export interface CalibrationTestMetrics extends CalibrationMetrics {
   confusion_matrix: ConfusionMatrix;
 }
 
-export type CalibrationStatus = "completed" | "failed_gates";
+export type CalibrationStatus = "running" | "completed" | "failed_gates";
 export type CalibrationFailureReason =
   | "recall_precondition_failed"
   | "precision_floor_unreachable"
@@ -330,6 +330,12 @@ export type CalibrationFailureReason =
   | "insufficient_sample"
   | "test_certification_failed";
 
+export interface TuneGateSlack {
+  auto_match_precision_lower_bound_slack: number;
+  auto_match_count_slack: number;
+  delivered_shortlist_recall_slack: number;
+}
+
 export interface CalibrationResult {
   status: CalibrationStatus;
   failed_reason?: CalibrationFailureReason;
@@ -337,6 +343,7 @@ export interface CalibrationResult {
   selected_thresholds?: SelectedThresholds;
   tune_metrics?: CalibrationMetrics;
   test_metrics?: CalibrationTestMetrics;
+  tune_gate_slack?: TuneGateSlack;
 }
 
 export interface RunCalibrationOptions {
@@ -357,7 +364,24 @@ export interface RunCalibrationOptions {
   minDeliveredShortlistRecallAtK?: number;
   /** Default: 30 */
   minAutoMatchCount?: number;
+  /** Tune-only selection buffer for Wilson auto-match precision lower bound. Default: 0.03 */
+  tuneAutoMatchPrecisionBuffer?: number;
+  /** Tune-only selection buffer for minimum auto-match count. Default: 3 */
+  tuneAutoMatchCountBuffer?: number;
+  /** Tune-only selection buffer for delivered shortlist recall. Default: 0.02 */
+  tuneDeliveredShortlistRecallBuffer?: number;
   candidateLimit: number;
+  /** Default: 4 */
+  concurrency?: number;
+  initialObservations?: Map<number, QueryObservation> | QueryObservation[];
+  onProgress?: (completed: number, total: number) => void;
+  onObservation?: (
+    observation: QueryObservation,
+    caseIndex: number,
+    completedCount: number,
+    totalCount: number,
+  ) => void | Promise<void>;
+  onObservationsReady?: () => void;
 }
 
 // ---------------------------------------------------------------------------
@@ -454,6 +478,97 @@ export function wilsonLowerBound(successes: number, total: number): number {
   return Math.max(0, (centre - adjustment) / denominator);
 }
 
+/**
+ * Computes the maximum attainable 95% Wilson lower bound on auto-match precision
+ * for the tune split subject to minAutoMatchCount and the number of matched tune cases.
+ * Under perfect ranking/classification (zero false positives), at most all true matched
+ * tune cases can be auto-matched.
+ */
+export function computeMaxAttainablePrecisionLowerBound(
+  cases: DecisionCase[],
+  minAutoMatchCount: number,
+): { tuneMatchedCount: number; maxAttainablePrecision: number } {
+  const tuneMatchedCount = cases.filter(
+    (c) => c.split === "tune" && c.expected_outcome === "matched",
+  ).length;
+  const effectiveTrials = Math.max(tuneMatchedCount, minAutoMatchCount);
+  const maxAttainablePrecision = wilsonLowerBound(tuneMatchedCount, effectiveTrials);
+  return { tuneMatchedCount, maxAttainablePrecision };
+}
+
+/**
+ * Asserts that the requested auto-match precision and count gates are mathematically
+ * attainable on the given dataset's tune split before inference begins.
+ */
+export function assertCalibrationFeasibility(
+  cases: DecisionCase[],
+  gates: {
+    minAutoMatchPrecision: number;
+    minAutoMatchCount: number;
+    minDeliveredShortlistRecallAtK?: number;
+    tuneAutoMatchPrecisionBuffer?: number;
+    tuneAutoMatchCountBuffer?: number;
+    tuneDeliveredShortlistRecallBuffer?: number;
+  },
+): void {
+  const precisionBuffer = gates.tuneAutoMatchPrecisionBuffer ?? 0.03;
+  const countBuffer = gates.tuneAutoMatchCountBuffer ?? 3;
+  const recallBuffer = gates.tuneDeliveredShortlistRecallBuffer ?? 0.02;
+
+  if (precisionBuffer < 0 || !Number.isFinite(precisionBuffer)) {
+    throw new Error("tune_auto_match_precision_buffer must be a non-negative number");
+  }
+  if (countBuffer < 0 || !Number.isInteger(countBuffer)) {
+    throw new Error("tune_auto_match_count_buffer must be a non-negative integer");
+  }
+  if (recallBuffer < 0 || !Number.isFinite(recallBuffer)) {
+    throw new Error("tune_delivered_shortlist_recall_buffer must be a non-negative number");
+  }
+
+  const effectivePrecision = gates.minAutoMatchPrecision + precisionBuffer;
+  if (effectivePrecision > 1.0) {
+    throw new Error(
+      `Requested calibration gates with tune buffers are mathematically impossible: ` +
+      `effective min_auto_match_precision (${effectivePrecision.toFixed(2)}) exceeds 1.0 ` +
+      `(min_auto_match_precision=${gates.minAutoMatchPrecision}, tune_buffer=${precisionBuffer}). ` +
+      `Lower --min-auto-match-precision or --tune-auto-match-precision-buffer.`,
+    );
+  }
+
+  const baseDeliveredRecall = gates.minDeliveredShortlistRecallAtK ?? 0.95;
+  const effectiveDeliveredRecall = baseDeliveredRecall + recallBuffer;
+  if (effectiveDeliveredRecall > 1.0) {
+    throw new Error(
+      `Requested calibration gates with tune buffers are mathematically impossible: ` +
+      `effective min_delivered_shortlist_recall_at_k (${effectiveDeliveredRecall.toFixed(2)}) exceeds 1.0 ` +
+      `(min_delivered_shortlist_recall_at_k=${baseDeliveredRecall}, tune_buffer=${recallBuffer}). ` +
+      `Lower --min-delivered-shortlist-recall-at-k or --tune-delivered-shortlist-recall-buffer.`,
+    );
+  }
+
+  const effectiveCount = gates.minAutoMatchCount + countBuffer;
+  const { tuneMatchedCount, maxAttainablePrecision } =
+    computeMaxAttainablePrecisionLowerBound(cases, effectiveCount);
+  if (effectiveCount > tuneMatchedCount) {
+    throw new Error(
+      `Requested calibration gates are mathematically unattainable on this dataset: ` +
+      `effective min_auto_match_count (${effectiveCount}) exceeds the number of matched cases in the ` +
+      `tune split (${tuneMatchedCount}) (min_auto_match_count=${gates.minAutoMatchCount}, tune_buffer=${countBuffer}). ` +
+      `Lower --min-auto-match-count or --tune-auto-match-count-buffer, or supply a dataset with more matched tune cases.`,
+    );
+  }
+  if (maxAttainablePrecision < effectivePrecision) {
+    throw new Error(
+      `Requested calibration gates are mathematically unattainable on this dataset: ` +
+      `min_auto_match_precision=${gates.minAutoMatchPrecision} requires more evidence than the ` +
+      `tune split provides (${tuneMatchedCount} matched cases, min_auto_match_count=${gates.minAutoMatchCount}, effective min_auto_match_count=${effectiveCount}). ` +
+      `The maximum attainable 95% Wilson lower bound under perfect classification is ` +
+      `${maxAttainablePrecision.toFixed(4)}. ` +
+      `Lower --min-auto-match-precision or supply a larger dataset.`,
+    );
+  }
+}
+
 function computeTestMetrics(
   observations: QueryObservation[],
   thresholds: SelectedThresholds,
@@ -741,6 +856,40 @@ function selectThresholds(
 // Public API — runCalibration (AC2, AC3, AC4)
 // ---------------------------------------------------------------------------
 
+async function processWithConcurrency<T>(
+  items: readonly T[],
+  concurrency: number,
+  processItem: (item: T) => Promise<void>,
+): Promise<void> {
+  let nextItemPosition = 0;
+  let firstError: unknown;
+
+  const claimNextPosition = (): number | undefined => {
+    if (firstError !== undefined || nextItemPosition >= items.length) return undefined;
+    return nextItemPosition++;
+  };
+
+  const workerCount = Math.min(concurrency, items.length);
+  const workers = Array.from({ length: workerCount }, async () => {
+    for (
+      let itemPosition = claimNextPosition();
+      itemPosition !== undefined;
+      itemPosition = claimNextPosition()
+    ) {
+      try {
+        await processItem(items[itemPosition]!);
+      } catch (error) {
+        firstError ??= error;
+      }
+    }
+  });
+
+  // Drain work already in flight before propagating an error. Callers may close
+  // resources after rejection, so no worker may still be checkpointing then.
+  await Promise.all(workers);
+  if (firstError !== undefined) throw firstError;
+}
+
 /**
  * Run an in-memory calibration:
  *  1. Require a configured reranker (AC2)
@@ -754,13 +903,24 @@ export async function runCalibration(opts: RunCalibrationOptions): Promise<Calib
     getCandidates,
     getRankedCandidates,
     reranker,
-    minAutoMatchPrecision = 0.99,
+    minAutoMatchPrecision = 0.75,
     minRetrievalRecallAtK = 0.95,
     minDeliveredShortlistRecallAtK = minRetrievalRecallAtK,
-    minAutoMatchCount = 30,
+    minAutoMatchCount = 15,
+    tuneAutoMatchPrecisionBuffer = 0.03,
+    tuneAutoMatchCountBuffer = 3,
+    tuneDeliveredShortlistRecallBuffer = 0.02,
     candidateLimit,
+    concurrency = 4,
+    initialObservations,
+    onProgress,
+    onObservation,
   } = opts;
 
+  if (!Number.isInteger(concurrency) || concurrency < 1) {
+    throw new Error("concurrency must be a positive integer");
+  }
+
   if (!getRankedCandidates && (!getCandidates || !reranker)) {
     throw new Error(
       "A configured reranker is required to run calibration. " +
@@ -769,8 +929,31 @@ export async function runCalibration(opts: RunCalibrationOptions): Promise<Calib
   }
 
   // --- Step 1: Cache observations (reranker called exactly once per query) ---
-  const observations: QueryObservation[] = [];
-  for (const c of cases) {
+  const obsMap = new Map<number, QueryObservation>();
+  if (initialObservations) {
+    if (Array.isArray(initialObservations)) {
+      initialObservations.forEach((obs, idx) => {
+        if (obs) obsMap.set(idx, obs);
+      });
+    } else {
+      for (const [idx, obs] of initialObservations.entries()) {
+        obsMap.set(idx, obs);
+      }
+    }
+  }
+
+  let completedCount = obsMap.size;
+  if (onProgress && completedCount > 0) {
+    onProgress(completedCount, cases.length);
+  }
+
+  const pendingIndices: number[] = [];
+  for (let i = 0; i < cases.length; i++) {
+    if (!obsMap.has(i)) pendingIndices.push(i);
+  }
+
+  await processWithConcurrency(pendingIndices, concurrency, async (caseIndex) => {
+    const c = cases[caseIndex]!;
     let ranked: Array<{ skill_id: string; score: number }>;
     if (getRankedCandidates) {
       ranked = await getRankedCandidates(c.query);
@@ -781,14 +964,25 @@ export async function runCalibration(opts: RunCalibrationOptions): Promise<Calib
         .map((d, i) => ({ skill_id: d.skill_id, score: scores[i] ?? 0 }))
         .sort((a, b) => b.score - a.score);
     }
-    observations.push({
+    const obs: QueryObservation = {
       query: c.query,
       split: c.split,
       expected_outcome: c.expected_outcome,
       relevant_skill_ids: c.relevant_skill_ids,
       ranked,
-    });
-  }
+    };
+    obsMap.set(caseIndex, obs);
+    if (onObservation) {
+      await onObservation(obs, caseIndex, completedCount + 1, cases.length);
+    }
+    const progressCount = ++completedCount;
+    if (onProgress) {
+      onProgress(progressCount, cases.length);
+    }
+  });
+
+  const observations: QueryObservation[] = cases.map((_, i) => obsMap.get(i)!);
+  opts.onObservationsReady?.();
 
   // --- Step 2: Select thresholds from tune split only ---
   const tuneObs = observations.filter((o) => o.split === "tune");
@@ -817,9 +1011,16 @@ export async function runCalibration(opts: RunCalibrationOptions): Promise<Calib
     minDeliveredShortlistRecallAtK,
     minAutoMatchCount,
   };
+  const tuneGates = {
+    minAutoMatchPrecision: minAutoMatchPrecision + tuneAutoMatchPrecisionBuffer,
+    minRetrievalRecallAtK,
+    minDeliveredShortlistRecallAtK:
+      minDeliveredShortlistRecallAtK + tuneDeliveredShortlistRecallBuffer,
+    minAutoMatchCount: minAutoMatchCount + tuneAutoMatchCountBuffer,
+  };
   const selection = selectThresholds(
     tuneObs,
-    gates,
+    tuneGates,
     candidateLimit,
   );
 
@@ -832,8 +1033,16 @@ export async function runCalibration(opts: RunCalibrationOptions): Promise<Calib
   }
   const selected = selection.selected;
 
-  // --- Step 3: Report tune metrics ---
+  // --- Step 3: Report tune metrics & explicit tune gate slack ---
   const tune_metrics = computeMetrics(tuneObs, selected, candidateLimit);
+  const tune_gate_slack: TuneGateSlack = {
+    auto_match_precision_lower_bound_slack:
+      tune_metrics.auto_match_precision_lower_bound - minAutoMatchPrecision,
+    auto_match_count_slack:
+      tune_metrics.auto_match_count - minAutoMatchCount,
+    delivered_shortlist_recall_slack:
+      tune_metrics.delivered_shortlist_recall_at_k - minDeliveredShortlistRecallAtK,
+  };
 
   // --- Step 4: Evaluate untouched test split ---
   const test_metrics = computeTestMetrics(testObs, selected, candidateLimit);
@@ -845,10 +1054,18 @@ export async function runCalibration(opts: RunCalibrationOptions): Promise<Calib
       selected_thresholds: selected,
       tune_metrics,
       test_metrics,
+      tune_gate_slack,
     };
   }
 
-  return { status: "completed", observations, selected_thresholds: selected, tune_metrics, test_metrics };
+  return {
+    status: "completed",
+    observations,
+    selected_thresholds: selected,
+    tune_metrics,
+    test_metrics,
+    tune_gate_slack,
+  };
 }
 
 // ---------------------------------------------------------------------------
@@ -879,6 +1096,10 @@ export interface CalibrationRunRecord {
   min_auto_match_count?: number;
   min_delivered_shortlist_recall_at_k?: number;
   min_shortlist_recall_at_5: number;
+  tune_auto_match_precision_buffer?: number;
+  tune_auto_match_count_buffer?: number;
+  tune_delivered_shortlist_recall_buffer?: number;
+  tune_gate_slack?: TuneGateSlack;
   failed_reason?: CalibrationFailureReason;
   selected_thresholds?: SelectedThresholds;
   tune_metrics?: CalibrationMetrics;
@@ -903,6 +1124,9 @@ export interface CalibrationRunSummary {
   min_auto_match_count: number;
   min_delivered_shortlist_recall_at_k: number;
   min_shortlist_recall_at_5: number;
+  tune_auto_match_precision_buffer: number;
+  tune_auto_match_count_buffer: number;
+  tune_delivered_shortlist_recall_buffer: number;
   failed_reason?: CalibrationFailureReason;
 }
 
@@ -928,10 +1152,62 @@ export function openCalibrateDb(stateDir: string): Database {
   const db = new Database(join(stateDir, "calibrate.sqlite3"), { create: true });
   db.run("PRAGMA journal_mode = WAL");
   db.run("PRAGMA busy_timeout = 2000");
+
+  const tableDef = db
+    .query("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'calibration_runs'")
+    .get() as { sql: string } | null;
+
+  if (
+    tableDef &&
+    tableDef.sql &&
+    tableDef.sql.includes("CHECK") &&
+    !tableDef.sql.includes("'running'")
+  ) {
+    const cols = db.query("PRAGMA table_info(calibration_runs)").all() as Array<{ name: string }>;
+    const colNames = cols.map((c) => c.name);
+    const selectCols = colNames.join(", ");
+
+    db.transaction(() => {
+      db.run(`CREATE TABLE calibration_runs_new (
+        run_id TEXT PRIMARY KEY,
+        created_at TEXT NOT NULL,
+        status TEXT NOT NULL CHECK (status IN ('running', 'completed', 'failed_gates')),
+        reranker_fingerprint TEXT NOT NULL,
+        embedding_fingerprint TEXT NOT NULL,
+        corpus_fingerprint TEXT NOT NULL,
+        dataset_hash TEXT NOT NULL,
+        min_auto_match_precision REAL NOT NULL,
+        min_shortlist_recall_at_5 REAL NOT NULL,
+        selected_thresholds TEXT,
+        tune_metrics TEXT,
+        test_metrics TEXT,
+        observations TEXT NOT NULL,
+        candidate_limit INTEGER NOT NULL DEFAULT 5,
+        attempt_count INTEGER NOT NULL DEFAULT 1,
+        min_auto_match_count INTEGER NOT NULL DEFAULT 1,
+        min_delivered_shortlist_recall_at_k REAL NOT NULL DEFAULT 0.95,
+        failed_reason TEXT,
+        dataset_provenance TEXT NOT NULL DEFAULT '{}',
+        human_labelled_case_count INTEGER NOT NULL DEFAULT 0,
+        imported_labelled_case_count INTEGER NOT NULL DEFAULT 0,
+        recall_settings TEXT NOT NULL DEFAULT '{}',
+        tune_auto_match_precision_buffer REAL NOT NULL DEFAULT 0.03,
+        tune_auto_match_count_buffer INTEGER NOT NULL DEFAULT 3,
+        tune_delivered_shortlist_recall_buffer REAL NOT NULL DEFAULT 0.02,
+        tune_gate_slack TEXT
+      )`);
+      db.run(
+        `INSERT INTO calibration_runs_new (${selectCols}) SELECT ${selectCols} FROM calibration_runs`,
+      );
+      db.run("DROP TABLE calibration_runs");
+      db.run("ALTER TABLE calibration_runs_new RENAME TO calibration_runs");
+    })();
+  }
+
   db.run(`CREATE TABLE IF NOT EXISTS calibration_runs (
     run_id TEXT PRIMARY KEY,
     created_at TEXT NOT NULL,
-    status TEXT NOT NULL CHECK (status IN ('completed', 'failed_gates')),
+    status TEXT NOT NULL CHECK (status IN ('running', 'completed', 'failed_gates')),
     reranker_fingerprint TEXT NOT NULL,
     embedding_fingerprint TEXT NOT NULL,
     corpus_fingerprint TEXT NOT NULL,
@@ -971,11 +1247,63 @@ export function openCalibrateDb(stateDir: string): Database {
   if (!columns.some((column) => column.name === "recall_settings")) {
     db.run("ALTER TABLE calibration_runs ADD COLUMN recall_settings TEXT NOT NULL DEFAULT '{}'");
   }
+  if (!columns.some((column) => column.name === "tune_auto_match_precision_buffer")) {
+    db.run("ALTER TABLE calibration_runs ADD COLUMN tune_auto_match_precision_buffer REAL NOT NULL DEFAULT 0.03");
+  }
+  if (!columns.some((column) => column.name === "tune_auto_match_count_buffer")) {
+    db.run("ALTER TABLE calibration_runs ADD COLUMN tune_auto_match_count_buffer INTEGER NOT NULL DEFAULT 3");
+  }
+  if (!columns.some((column) => column.name === "tune_delivered_shortlist_recall_buffer")) {
+    db.run("ALTER TABLE calibration_runs ADD COLUMN tune_delivered_shortlist_recall_buffer REAL NOT NULL DEFAULT 0.02");
+  }
+  if (!columns.some((column) => column.name === "tune_gate_slack")) {
+    db.run("ALTER TABLE calibration_runs ADD COLUMN tune_gate_slack TEXT");
+  }
+
+  db.run(`CREATE TABLE IF NOT EXISTS calibration_observations (
+    run_id TEXT NOT NULL,
+    case_index INTEGER NOT NULL,
+    query TEXT NOT NULL,
+    split TEXT NOT NULL,
+    expected_outcome TEXT NOT NULL,
+    relevant_skill_ids TEXT NOT NULL,
+    ranked TEXT NOT NULL,
+    PRIMARY KEY (run_id, case_index),
+    FOREIGN KEY (run_id) REFERENCES calibration_runs(run_id) ON DELETE CASCADE
+  )`);
+
   return db;
 }
 
-/** Persist a calibration run (all fields) to the evidence store. */
-export function insertCalibrationRun(db: Database, run: CalibrationRunRecord): void {
+export interface CreateInitialCalibrationRunOptions {
+  run_id: string;
+  created_at: string;
+  status: "running";
+  reranker_fingerprint: string;
+  embedding_fingerprint: string;
+  corpus_fingerprint: string;
+  dataset_hash: string;
+  candidate_limit: number;
+  min_auto_match_precision: number;
+  min_auto_match_count?: number;
+  min_delivered_shortlist_recall_at_k?: number;
+  min_shortlist_recall_at_5: number;
+  tune_auto_match_precision_buffer?: number;
+  tune_auto_match_count_buffer?: number;
+  tune_delivered_shortlist_recall_buffer?: number;
+  dataset_provenance?: DatasetProvenanceSummary;
+  recall_settings?: {
+    k_lexical: number;
+    k_vector: number;
+    k_rerank: number;
+  };
+}
+
+/** Create an initial calibration run record with 'running' status before inference starts. */
+export function createInitialCalibrationRun(
+  db: Database,
+  run: CreateInitialCalibrationRunOptions,
+): void {
   const attemptCount = (
     db.query("SELECT COUNT(*) AS count FROM calibration_runs WHERE dataset_hash = ?")
       .get(run.dataset_hash) as { count: number }
@@ -989,8 +1317,10 @@ export function insertCalibrationRun(db: Database, run: CalibrationRunRecord): v
       min_delivered_shortlist_recall_at_k, min_shortlist_recall_at_5, failed_reason,
       selected_thresholds, tune_metrics, test_metrics, observations,
       dataset_provenance, human_labelled_case_count, imported_labelled_case_count,
-      recall_settings
-    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      recall_settings,
+      tune_auto_match_precision_buffer, tune_auto_match_count_buffer,
+      tune_delivered_shortlist_recall_buffer, tune_gate_slack
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
     [
       run.run_id,
       run.created_at,
@@ -1005,19 +1335,174 @@ export function insertCalibrationRun(db: Database, run: CalibrationRunRecord): v
       run.min_auto_match_count ?? 1,
       run.min_delivered_shortlist_recall_at_k ?? run.min_shortlist_recall_at_5,
       run.min_shortlist_recall_at_5,
+      null,
+      null,
+      null,
+      null,
+      "[]",
+      JSON.stringify(run.dataset_provenance ?? {}),
+      run.dataset_provenance?.human_labelled_case_count ?? 0,
+      run.dataset_provenance?.imported_labelled_case_count ?? 0,
+      JSON.stringify(run.recall_settings ?? {}),
+      run.tune_auto_match_precision_buffer ?? 0.03,
+      run.tune_auto_match_count_buffer ?? 3,
+      run.tune_delivered_shortlist_recall_buffer ?? 0.02,
+      null,
+    ],
+  );
+}
+
+/** Save a single case observation incrementally. */
+export function saveCalibrationObservation(
+  db: Database,
+  runId: string,
+  caseIndex: number,
+  observation: QueryObservation,
+): void {
+  db.run(
+    `INSERT OR REPLACE INTO calibration_observations (
+      run_id, case_index, query, split, expected_outcome, relevant_skill_ids, ranked
+    ) VALUES (?, ?, ?, ?, ?, ?, ?)`,
+    [
+      runId,
+      caseIndex,
+      observation.query,
+      observation.split,
+      observation.expected_outcome,
+      JSON.stringify(observation.relevant_skill_ids),
+      JSON.stringify(observation.ranked),
+    ],
+  );
+}
+
+/** Retrieve all completed observations for a run_id, keyed by case index. */
+export function getCalibrationObservations(
+  db: Database,
+  runId: string,
+): Map<number, QueryObservation> {
+  const rows = db
+    .query(
+      `SELECT case_index, query, split, expected_outcome, relevant_skill_ids, ranked
+       FROM calibration_observations WHERE run_id = ? ORDER BY case_index ASC`,
+    )
+    .all(runId) as Array<{
+      case_index: number;
+      query: string;
+      split: DecisionSplit;
+      expected_outcome: DecisionOutcome;
+      relevant_skill_ids: string;
+      ranked: string;
+    }>;
+  const map = new Map<number, QueryObservation>();
+  for (const r of rows) {
+    map.set(r.case_index, {
+      query: r.query,
+      split: r.split,
+      expected_outcome: r.expected_outcome,
+      relevant_skill_ids: JSON.parse(r.relevant_skill_ids) as string[],
+      ranked: JSON.parse(r.ranked) as Array<{ skill_id: string; score: number }>,
+    });
+  }
+  return map;
+}
+
+export interface FinalizeCalibrationRunOptions {
+  run_id: string;
+  status: CalibrationStatus;
+  failed_reason?: CalibrationFailureReason;
+  selected_thresholds?: SelectedThresholds;
+  tune_metrics?: CalibrationMetrics;
+  test_metrics?: CalibrationTestMetrics;
+  tune_gate_slack?: TuneGateSlack;
+  observations: QueryObservation[];
+}
+
+/** Finalize a calibration run record with its completion/failure status, metrics, and thresholds. */
+export function finalizeCalibrationRun(
+  db: Database,
+  run: FinalizeCalibrationRunOptions,
+): void {
+  db.run(
+    `UPDATE calibration_runs SET
+      status = ?,
+      failed_reason = ?,
+      selected_thresholds = ?,
+      tune_metrics = ?,
+      test_metrics = ?,
+      observations = ?,
+      tune_gate_slack = ?
+     WHERE run_id = ?`,
+    [
+      run.status,
       run.failed_reason ?? null,
       run.selected_thresholds != null ? JSON.stringify(run.selected_thresholds) : null,
       run.tune_metrics != null ? JSON.stringify(run.tune_metrics) : null,
       run.test_metrics != null ? JSON.stringify(run.test_metrics) : null,
       JSON.stringify(run.observations),
-      JSON.stringify(run.dataset_provenance ?? {}),
-      run.dataset_provenance?.human_labelled_case_count ?? 0,
-      run.dataset_provenance?.imported_labelled_case_count ?? 0,
-      JSON.stringify(run.recall_settings ?? {}),
+      run.tune_gate_slack != null ? JSON.stringify(run.tune_gate_slack) : null,
+      run.run_id,
     ],
   );
 }
 
+/** Persist a calibration run (all fields) to the evidence store. */
+export function insertCalibrationRun(db: Database, run: CalibrationRunRecord): void {
+  const attemptCount = (
+    db.query("SELECT COUNT(*) AS count FROM calibration_runs WHERE dataset_hash = ?")
+      .get(run.dataset_hash) as { count: number }
+  ).count + 1;
+  db.transaction(() => {
+    db.run(
+      `INSERT OR REPLACE INTO calibration_runs (
+        run_id, created_at, status,
+        reranker_fingerprint, embedding_fingerprint, corpus_fingerprint, dataset_hash,
+        candidate_limit,
+        attempt_count, min_auto_match_precision, min_auto_match_count,
+        min_delivered_shortlist_recall_at_k, min_shortlist_recall_at_5, failed_reason,
+        selected_thresholds, tune_metrics, test_metrics, observations,
+        dataset_provenance, human_labelled_case_count, imported_labelled_case_count,
+        recall_settings,
+        tune_auto_match_precision_buffer, tune_auto_match_count_buffer,
+        tune_delivered_shortlist_recall_buffer, tune_gate_slack
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      [
+        run.run_id,
+        run.created_at,
+        run.status,
+        run.reranker_fingerprint,
+        run.embedding_fingerprint,
+        run.corpus_fingerprint,
+        run.dataset_hash,
+        run.candidate_limit,
+        attemptCount,
+        run.min_auto_match_precision,
+        run.min_auto_match_count ?? 1,
+        run.min_delivered_shortlist_recall_at_k ?? run.min_shortlist_recall_at_5,
+        run.min_shortlist_recall_at_5,
+        run.failed_reason ?? null,
+        run.selected_thresholds != null ? JSON.stringify(run.selected_thresholds) : null,
+        run.tune_metrics != null ? JSON.stringify(run.tune_metrics) : null,
+        run.test_metrics != null ? JSON.stringify(run.test_metrics) : null,
+        JSON.stringify(run.observations),
+        JSON.stringify(run.dataset_provenance ?? {}),
+        run.dataset_provenance?.human_labelled_case_count ?? 0,
+        run.dataset_provenance?.imported_labelled_case_count ?? 0,
+        JSON.stringify(run.recall_settings ?? {}),
+        run.tune_auto_match_precision_buffer ?? 0.03,
+        run.tune_auto_match_count_buffer ?? 3,
+        run.tune_delivered_shortlist_recall_buffer ?? 0.02,
+        run.tune_gate_slack != null ? JSON.stringify(run.tune_gate_slack) : null,
+      ],
+    );
+    if (run.observations && run.observations.length > 0) {
+      for (let i = 0; i < run.observations.length; i++) {
+        const obs = run.observations[i]!;
+        saveCalibrationObservation(db, run.run_id, i, obs);
+      }
+    }
+  })();
+}
+
 interface RawCalibrationRow {
   run_id: string;
   created_at: string;
@@ -1041,6 +1526,10 @@ interface RawCalibrationRow {
   human_labelled_case_count: number;
   imported_labelled_case_count: number;
   recall_settings: string;
+  tune_auto_match_precision_buffer: number | null;
+  tune_auto_match_count_buffer: number | null;
+  tune_delivered_shortlist_recall_buffer: number | null;
+  tune_gate_slack: string | null;
 }
 
 function parseMetrics(json: string): CalibrationMetrics {
@@ -1089,6 +1578,10 @@ function rowToRecord(row: RawCalibrationRow): CalibrationRunRecord {
     min_auto_match_count: row.min_auto_match_count,
     min_delivered_shortlist_recall_at_k: row.min_delivered_shortlist_recall_at_k,
     min_shortlist_recall_at_5: row.min_shortlist_recall_at_5,
+    tune_auto_match_precision_buffer: row.tune_auto_match_precision_buffer ?? 0.03,
+    tune_auto_match_count_buffer: row.tune_auto_match_count_buffer ?? 3,
+    tune_delivered_shortlist_recall_buffer: row.tune_delivered_shortlist_recall_buffer ?? 0.02,
+    tune_gate_slack: row.tune_gate_slack != null ? JSON.parse(row.tune_gate_slack) : undefined,
     failed_reason: row.failed_reason as CalibrationFailureReason | null ?? undefined,
     selected_thresholds: row.selected_thresholds != null
       ? (JSON.parse(row.selected_thresholds) as SelectedThresholds)
@@ -1111,7 +1604,16 @@ export function getCalibrationRun(db: Database, runId: string): CalibrationRunRe
   const row = db
     .query("SELECT * FROM calibration_runs WHERE run_id = ?")
     .get(runId) as RawCalibrationRow | null;
-  return row ? rowToRecord(row) : null;
+  if (!row) return null;
+  const record = rowToRecord(row);
+  if (record.observations.length === 0) {
+    const obsMap = getCalibrationObservations(db, runId);
+    if (obsMap.size > 0) {
+      const sortedIndices = Array.from(obsMap.keys()).sort((a, b) => a - b);
+      record.observations = sortedIndices.map((idx) => obsMap.get(idx)!);
+    }
+  }
+  return record;
 }
 
 /** List all runs ordered by created_at descending (excludes observations blob). */
@@ -1122,7 +1624,9 @@ export function listCalibrationRuns(db: Database): CalibrationRunSummary[] {
         reranker_fingerprint, embedding_fingerprint, corpus_fingerprint, dataset_hash,
         candidate_limit,
         attempt_count, min_auto_match_precision, min_auto_match_count,
-        min_delivered_shortlist_recall_at_k, min_shortlist_recall_at_5, failed_reason,
+        min_delivered_shortlist_recall_at_k, min_shortlist_recall_at_5,
+        tune_auto_match_precision_buffer, tune_auto_match_count_buffer,
+        tune_delivered_shortlist_recall_buffer, failed_reason,
         human_labelled_case_count, imported_labelled_case_count
        FROM calibration_runs ORDER BY created_at DESC`,
     )