@klhapp/skillmux 1.0.1 → 1.2.0

package/src/calibrate.ts

@@ -2,6 +2,8 @@ import { mkdirSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 import { Database } from "bun:sqlite";
 import { z } from "zod";
+import { decideResolveOutcome } from "./decision";
+import type { AuditRow, RankedCandidate } from "./types";
 
 export { generateDataset, type GenerateDatasetOptions } from "./dataset-generator";
 
@@ -13,11 +15,21 @@ export { generateDataset, type GenerateDatasetOptions } from "./dataset-generato
 export type DecisionSplit = "tune" | "test";
 export type DecisionOutcome = "matched" | "ambiguous" | "no_match";
 
+export interface DecisionCaseProvenance {
+  version: 1;
+  source: "authored" | "audit_import";
+  review_status: "human_labelled" | "unreviewed";
+  query_storage: "raw" | "redacted";
+  audit_id?: number;
+  labelled_at?: string;
+}
+
 export interface DecisionCase {
   query: string;
   split: DecisionSplit;
   expected_outcome: DecisionOutcome;
   relevant_skill_ids: string[];
+  provenance?: DecisionCaseProvenance;
 }
 
 // ---------------------------------------------------------------------------
@@ -29,6 +41,14 @@ const rawCaseSchema = z.object({
   split: z.enum(["tune", "test"]),
   expected_outcome: z.enum(["matched", "ambiguous", "no_match"]),
   relevant_skill_ids: z.array(z.string()),
+  provenance: z.object({
+    version: z.literal(1),
+    source: z.enum(["authored", "audit_import"]),
+    review_status: z.enum(["human_labelled", "unreviewed"]),
+    query_storage: z.enum(["raw", "redacted"]),
+    audit_id: z.number().int().positive().optional(),
+    labelled_at: z.string().datetime().optional(),
+  }).strict().optional(),
 }).strict();
 
 type RawCase = z.infer<typeof rawCaseSchema>;
@@ -43,6 +63,25 @@ function validateCase(raw: RawCase, idx: number): DecisionCase {
   }
 
   const { expected_outcome, relevant_skill_ids } = raw;
+  const provenance: DecisionCaseProvenance = raw.provenance ?? {
+    version: 1,
+    source: "authored",
+    review_status: "human_labelled",
+    query_storage: "raw",
+  };
+
+  if (provenance.source === "audit_import") {
+    if (provenance.audit_id === undefined) {
+      throw new Error(
+        `Validation error at case ${idx}: imported field "provenance.audit_id" is required`,
+      );
+    }
+    if (provenance.review_status !== "human_labelled" || !provenance.labelled_at) {
+      throw new Error(
+        `Validation error at case ${idx}: imported audit case is unreviewed; human label and "provenance.labelled_at" are required for certification`,
+      );
+    }
+  }
 
   if (expected_outcome === "matched") {
     if (relevant_skill_ids.length !== 1) {
@@ -65,7 +104,85 @@ function validateCase(raw: RawCase, idx: number): DecisionCase {
     }
   }
 
-  return raw as DecisionCase;
+  return { ...raw, provenance };
+}
+
+export interface AuditFeedbackLabel {
+  split: DecisionSplit;
+  expected_outcome: DecisionOutcome;
+  relevant_skill_ids: string[];
+  labelled_at: string;
+}
+
+export type AuditQueryPrivacy =
+  | { include_raw_query: true }
+  | { include_raw_query: false; redacted_query: string };
+
+/** Import an audit outcome only after a separate human label is supplied. */
+export function importLabelledAuditCase(
+  audit: AuditRow,
+  label: AuditFeedbackLabel,
+  privacy: AuditQueryPrivacy,
+): DecisionCase {
+  const query = privacy.include_raw_query ? audit.query : privacy.redacted_query.trim();
+  if (!query) {
+    throw new Error("A non-empty redacted_query is required when raw audit queries are excluded");
+  }
+  const parsed = rawCaseSchema.parse({
+    query,
+    split: label.split,
+    expected_outcome: label.expected_outcome,
+    relevant_skill_ids: label.relevant_skill_ids,
+    provenance: {
+      version: 1,
+      source: "audit_import",
+      review_status: "human_labelled",
+      query_storage: privacy.include_raw_query ? "raw" : "redacted",
+      audit_id: audit.id,
+      labelled_at: label.labelled_at,
+    },
+  });
+  return validateCase(parsed, audit.id);
+}
+
+export interface DatasetProvenanceSummary {
+  version: 1;
+  human_labelled_case_count: number;
+  imported_labelled_case_count: number;
+  imported_unreviewed_case_count: number;
+  raw_query_case_count: number;
+  redacted_query_case_count: number;
+}
+
+export function summarizeDatasetProvenance(
+  cases: DecisionCase[],
+): DatasetProvenanceSummary {
+  const provenance = (item: DecisionCase): DecisionCaseProvenance =>
+    item.provenance ?? {
+      version: 1,
+      source: "authored",
+      review_status: "human_labelled",
+      query_storage: "raw",
+    };
+  return {
+    version: 1,
+    human_labelled_case_count:
+      cases.filter((item) => provenance(item).review_status === "human_labelled").length,
+    imported_labelled_case_count:
+      cases.filter((item) =>
+        provenance(item).source === "audit_import" &&
+        provenance(item).review_status === "human_labelled"
+      ).length,
+    imported_unreviewed_case_count:
+      cases.filter((item) =>
+        provenance(item).source === "audit_import" &&
+        provenance(item).review_status === "unreviewed"
+      ).length,
+    raw_query_case_count:
+      cases.filter((item) => provenance(item).query_storage === "raw").length,
+    redacted_query_case_count:
+      cases.filter((item) => provenance(item).query_storage === "redacted").length,
+  };
 }
 
 // ---------------------------------------------------------------------------
@@ -111,8 +228,12 @@ function validateDatasetCompleteness(cases: DecisionCase[]): void {
  *     no_match → 0)
  *   - Dataset completeness (both splits, all outcome types in each split)
  */
-export function loadDecisionCases(raw: unknown[]): DecisionCase[] {
+export function loadDecisionCases(
+  raw: unknown[],
+  validSkillIds?: Iterable<string>,
+): DecisionCase[] {
   const parsed: DecisionCase[] = [];
+  const validIds = validSkillIds ? new Set(validSkillIds) : undefined;
 
   for (let i = 0; i < raw.length; i++) {
     const item = raw[i];
@@ -126,7 +247,17 @@ export function loadDecisionCases(raw: unknown[]): DecisionCase[] {
       );
     }
 
-    parsed.push(validateCase(result.data, i));
+    const parsedCase = validateCase(result.data, i);
+    if (validIds) {
+      for (const skillId of parsedCase.relevant_skill_ids) {
+        if (!validIds.has(skillId)) {
+          throw new Error(
+            `Validation error at case ${i}: field "relevant_skill_ids" references unknown vault skill "${skillId}"`,
+          );
+        }
+      }
+    }
+    parsed.push(parsedCase);
   }
 
   validateDatasetCompleteness(parsed);
@@ -137,9 +268,12 @@ export function loadDecisionCases(raw: unknown[]): DecisionCase[] {
  * Read a JSON file from disk and validate it as a decision-policy dataset.
  * Throws if the file cannot be read or the contents fail validation.
  */
-export function loadDecisionCasesFromFile(path: string): DecisionCase[] {
+export function loadDecisionCasesFromFile(
+  path: string,
+  validSkillIds?: Iterable<string>,
+): DecisionCase[] {
   const raw = JSON.parse(readFileSync(path, "utf8")) as unknown[];
-  return loadDecisionCases(raw);
+  return loadDecisionCases(raw, validSkillIds);
 }
 
 // ---------------------------------------------------------------------------
@@ -169,9 +303,12 @@ export interface SelectedThresholds {
 
 export interface CalibrationMetrics {
   auto_match_precision: number;
+  auto_match_precision_lower_bound: number;
   auto_match_coverage: number;
-  shortlist_recall_at_5: number;
-  false_no_match_rate: number;
+  auto_match_count: number;
+  correct_auto_match_count: number;
+  retrieval_recall_at_k: number;
+  delivered_shortlist_recall_at_k: number;
 }
 
 export interface ConfusionMatrix {
@@ -185,9 +322,16 @@ export interface CalibrationTestMetrics extends CalibrationMetrics {
 }
 
 export type CalibrationStatus = "completed" | "failed_gates";
+export type CalibrationFailureReason =
+  | "recall_precondition_failed"
+  | "precision_floor_unreachable"
+  | "no_coverage"
+  | "insufficient_sample"
+  | "test_certification_failed";
 
 export interface CalibrationResult {
   status: CalibrationStatus;
+  failed_reason?: CalibrationFailureReason;
   observations: QueryObservation[];
   selected_thresholds?: SelectedThresholds;
   tune_metrics?: CalibrationMetrics;
@@ -196,87 +340,123 @@ export interface CalibrationResult {
 
 export interface RunCalibrationOptions {
   cases: DecisionCase[];
-  getCandidates: (query: string) => Promise<CandidateDoc[]>;
+  getCandidates?: (query: string) => Promise<CandidateDoc[]>;
+  /** Production hook: candidates already scored by the shared retrieval pipeline. */
+  getRankedCandidates?: (
+    query: string,
+  ) => Promise<Array<{ skill_id: string; score: number }>>;
   reranker:
     | ((query: string, docs: CandidateDoc[]) => Promise<number[]>)
     | undefined;
   /** Default: 0.99 */
   minAutoMatchPrecision?: number;
   /** Default: 0.95 */
-  minShortlistRecallAt5?: number;
+  minRetrievalRecallAtK?: number;
+  /** Default: minRetrievalRecallAtK */
+  minDeliveredShortlistRecallAtK?: number;
+  /** Default: 30 */
+  minAutoMatchCount?: number;
+  candidateLimit: number;
 }
 
 // ---------------------------------------------------------------------------
 // Decision simulation using cached observations
 // ---------------------------------------------------------------------------
 
-type SimulatedDecision = "matched" | "ambiguous" | "no_match";
-
-function simulateDecision(
+function decideObservation(
   obs: QueryObservation,
   thresholds: SelectedThresholds,
   candidateLimit: number,
-): SimulatedDecision {
-  const { match_score, match_margin, candidate_floor } = thresholds;
-  const eligible = obs.ranked.filter((c) => c.score >= candidate_floor);
-  if (eligible.length === 0) return "no_match";
-
-  const top = eligible[0]!;
-  const second = obs.ranked[1];
-  const margin = second ? top.score - second.score : top.score;
-
-  if (top.score >= match_score && margin >= match_margin) return "matched";
-  if (eligible.slice(0, candidateLimit).length > 0) return "ambiguous";
-  return "no_match";
+) {
+  const candidates: RankedCandidate[] = obs.ranked.map((candidate) => ({
+    ...candidate,
+    title: candidate.skill_id,
+    description: "",
+  }));
+  return decideResolveOutcome({
+    reranked: true,
+    candidates,
+    thresholds: { ...thresholds, candidate_limit: candidateLimit },
+  });
 }
 
 function computeMetrics(
   observations: QueryObservation[],
   thresholds: SelectedThresholds,
-  candidateLimit = 5,
+  candidateLimit: number,
 ): CalibrationMetrics {
   let autoMatchCount = 0;
   let correctAutoMatch = 0;
-  let shortlistHit = 0;
-  let falseNoMatch = 0;
+  let retrievalHit = 0;
+  let deliveredHit = 0;
   const matchableCases = observations.filter((o) => o.expected_outcome !== "no_match");
+  const expectedMatches = observations.filter((o) => o.expected_outcome === "matched");
 
   for (const obs of observations) {
-    const decision = simulateDecision(obs, thresholds, candidateLimit);
-    if (decision === "matched") {
+    const decision = decideObservation(obs, thresholds, candidateLimit);
+    if (decision.outcome === "matched") {
       autoMatchCount++;
-      // Correct if the top candidate is in relevant_skill_ids
       const top = obs.ranked[0];
-      if (top && obs.relevant_skill_ids.includes(top.skill_id)) correctAutoMatch++;
+      if (
+        obs.expected_outcome === "matched" &&
+        top?.skill_id === obs.relevant_skill_ids[0]
+      ) {
+        correctAutoMatch++;
+      }
     }
     if (obs.expected_outcome !== "no_match") {
-      // Shortlist recall: at least one relevant skill in top 5
-      const top5 = obs.ranked.slice(0, 5).map((c) => c.skill_id);
-      if (obs.relevant_skill_ids.some((id) => top5.includes(id))) shortlistHit++;
-    }
-    if (obs.expected_outcome !== "no_match" && decision === "no_match") {
-      falseNoMatch++;
+      const topK = obs.ranked.slice(0, candidateLimit).map((c) => c.skill_id);
+      if (obs.relevant_skill_ids.some((id) => topK.includes(id))) retrievalHit++;
+      const deliveredIds = decision.outcome === "matched"
+        ? [decision.skill_id]
+        : decision.outcome === "ambiguous"
+          ? decision.candidates.map((candidate) => candidate.skill_id)
+          : [];
+      if (obs.relevant_skill_ids.some((id) => deliveredIds.includes(id))) deliveredHit++;
     }
   }
 
-  const auto_match_precision = autoMatchCount === 0 ? 1.0 : correctAutoMatch / autoMatchCount;
-  const auto_match_coverage = matchableCases.length === 0
+  const auto_match_precision = autoMatchCount === 0 ? 0 : correctAutoMatch / autoMatchCount;
+  const auto_match_precision_lower_bound = wilsonLowerBound(correctAutoMatch, autoMatchCount);
+  const auto_match_coverage = expectedMatches.length === 0
     ? 0
-    : autoMatchCount / matchableCases.length;
-  const shortlist_recall_at_5 = matchableCases.length === 0
+    : correctAutoMatch / expectedMatches.length;
+  const retrieval_recall_at_k = matchableCases.length === 0
     ? 1.0
-    : shortlistHit / matchableCases.length;
-  const false_no_match_rate = matchableCases.length === 0
-    ? 0
-    : falseNoMatch / matchableCases.length;
+    : retrievalHit / matchableCases.length;
+  const delivered_shortlist_recall_at_k = matchableCases.length === 0
+    ? 1.0
+    : deliveredHit / matchableCases.length;
+
+  return {
+    auto_match_precision,
+    auto_match_precision_lower_bound,
+    auto_match_coverage,
+    auto_match_count: autoMatchCount,
+    correct_auto_match_count: correctAutoMatch,
+    retrieval_recall_at_k,
+    delivered_shortlist_recall_at_k,
+  };
+}
 
-  return { auto_match_precision, auto_match_coverage, shortlist_recall_at_5, false_no_match_rate };
+/** 95% Wilson score lower confidence bound for a binomial proportion. */
+export function wilsonLowerBound(successes: number, total: number): number {
+  if (total === 0) return 0;
+  const z = 1.959963984540054;
+  const proportion = successes / total;
+  const zSquared = z * z;
+  const denominator = 1 + zSquared / total;
+  const centre = proportion + zSquared / (2 * total);
+  const adjustment = z * Math.sqrt(
+    (proportion * (1 - proportion) + zSquared / (4 * total)) / total,
+  );
+  return Math.max(0, (centre - adjustment) / denominator);
 }
 
 function computeTestMetrics(
   observations: QueryObservation[],
   thresholds: SelectedThresholds,
-  candidateLimit = 5,
+  candidateLimit: number,
 ): CalibrationTestMetrics {
   const base = computeMetrics(observations, thresholds, candidateLimit);
 
@@ -285,8 +465,8 @@ function computeTestMetrics(
   const matrix: ConfusionMatrix = { matched: emptyRow(), ambiguous: emptyRow(), no_match: emptyRow() };
 
   for (const obs of observations) {
-    const predicted = simulateDecision(obs, thresholds, candidateLimit);
-    matrix[obs.expected_outcome][predicted]++;
+    const predicted = decideObservation(obs, thresholds, candidateLimit);
+    matrix[obs.expected_outcome][predicted.outcome]++;
   }
 
   return { ...base, confusion_matrix: matrix };
@@ -300,13 +480,26 @@ function uniqueSorted(values: number[]): number[] {
   return [...new Set(values)].sort((a, b) => a - b);
 }
 
+/** Smallest representable number greater than value. */
+function nextUp(value: number): number {
+  if (!Number.isFinite(value)) return value;
+  if (Object.is(value, -0)) value = 0;
+  const buffer = new ArrayBuffer(8);
+  const float = new Float64Array(buffer);
+  const bits = new BigUint64Array(buffer);
+  float[0] = value;
+  bits[0] = bits[0]! + (value >= 0 ? 1n : -1n);
+  return float[0]!;
+}
+
 function deriveThresholdCandidates(observations: QueryObservation[]): {
   scoreBreakpoints: number[];
   marginBreakpoints: number[];
   floorBreakpoints: number[];
 } {
-  const scores: number[] = [];
+  const scores: number[] = [0];
   const margins: number[] = [];
+  const floors: number[] = [0];
 
   for (const obs of observations) {
     if (obs.ranked.length === 0) continue;
@@ -314,22 +507,15 @@ function deriveThresholdCandidates(observations: QueryObservation[]): {
     scores.push(top.score);
     const second = obs.ranked[1];
     margins.push(second ? top.score - second.score : top.score);
+    // candidate_floor is inclusive, so the policy changes only immediately
+    // above an observed score. Use that transition to make every breakpoint
+    // capable of trimming at least one non-top candidate.
+    for (const candidate of obs.ranked.slice(1)) floors.push(nextUp(candidate.score));
   }
 
-  // Breakpoints: observed values + a small epsilon step below each
-  const epsilon = 0.001;
-  const scoreBreakpoints = uniqueSorted([
-    ...scores.map((s) => Math.max(0, s - epsilon)),
-    ...scores,
-  ]);
-  const marginBreakpoints = uniqueSorted([
-    ...margins.map((m) => Math.max(0, m - epsilon)),
-    ...margins,
-  ]);
-  const floorBreakpoints = uniqueSorted([
-    ...scores.map((s) => Math.max(0, s - epsilon)),
-    ...scores,
-  ]);
+  const scoreBreakpoints = uniqueSorted(scores);
+  const marginBreakpoints = uniqueSorted([0, ...margins]);
+  const floorBreakpoints = uniqueSorted(floors);
 
   return { scoreBreakpoints, marginBreakpoints, floorBreakpoints };
 }
@@ -340,61 +526,214 @@ function deriveThresholdCandidates(observations: QueryObservation[]): {
 
 /**
  * Find the threshold triple that:
- *   1. Satisfies minAutoMatchPrecision AND minShortlistRecallAt5 gates
+ *   1. Satisfies Wilson precision, sample-count, coverage, and delivered-recall gates
  *   2. Among those: maximizes auto_match_coverage
- *   3. Ties broken by: higher auto_match_precision, then higher shortlist_recall_at_5,
- *      then lower auto_match_coverage (as coverage the tiebreak)
+ *   3. Ties break on confidence, delivered recall, maximal safe shortlist trimming,
+ *      then deterministic lower match thresholds
  */
+interface CalibrationGates {
+  minAutoMatchPrecision: number;
+  minRetrievalRecallAtK: number;
+  minDeliveredShortlistRecallAtK: number;
+  minAutoMatchCount: number;
+}
+
+function metricsPass(metrics: CalibrationMetrics, gates: CalibrationGates): boolean {
+  return (
+    metrics.auto_match_precision_lower_bound >= gates.minAutoMatchPrecision &&
+    metrics.auto_match_count >= gates.minAutoMatchCount &&
+    metrics.auto_match_coverage > 0 &&
+    metrics.delivered_shortlist_recall_at_k >= gates.minDeliveredShortlistRecallAtK
+  );
+}
+
+function betterPolicy(
+  candidate: { thresholds: SelectedThresholds; metrics: CalibrationMetrics },
+  best: { thresholds: SelectedThresholds; metrics: CalibrationMetrics } | undefined,
+): boolean {
+  if (!best) return true;
+  const a = candidate.metrics;
+  const b = best.metrics;
+  if (a.auto_match_coverage !== b.auto_match_coverage) {
+    return a.auto_match_coverage > b.auto_match_coverage;
+  }
+  if (a.auto_match_precision_lower_bound !== b.auto_match_precision_lower_bound) {
+    return a.auto_match_precision_lower_bound > b.auto_match_precision_lower_bound;
+  }
+  if (a.delivered_shortlist_recall_at_k !== b.delivered_shortlist_recall_at_k) {
+    return a.delivered_shortlist_recall_at_k > b.delivered_shortlist_recall_at_k;
+  }
+  if (candidate.thresholds.candidate_floor !== best.thresholds.candidate_floor) {
+    return candidate.thresholds.candidate_floor > best.thresholds.candidate_floor;
+  }
+  if (candidate.thresholds.match_score !== best.thresholds.match_score) {
+    return candidate.thresholds.match_score < best.thresholds.match_score;
+  }
+  if (candidate.thresholds.match_margin !== best.thresholds.match_margin) {
+    return candidate.thresholds.match_margin < best.thresholds.match_margin;
+  }
+  return false;
+}
+
+function sampledFloorIndexes(length: number, maxSamples = 32): number[] {
+  if (length <= maxSamples) return Array.from({ length }, (_, index) => index);
+  return uniqueSorted(
+    Array.from({ length: maxSamples }, (_, index) =>
+      Math.round(index * (length - 1) / (maxSamples - 1))),
+  );
+}
+
 function selectThresholds(
   tuneObservations: QueryObservation[],
-  gates: { minAutoMatchPrecision: number; minShortlistRecallAt5: number },
+  gates: CalibrationGates,
   candidateLimit: number,
-): SelectedThresholds | undefined {
+): { selected?: SelectedThresholds; reason?: CalibrationFailureReason } {
   const { scoreBreakpoints, marginBreakpoints, floorBreakpoints } =
     deriveThresholdCandidates(tuneObservations);
 
   let best:
     | { thresholds: SelectedThresholds; metrics: CalibrationMetrics }
     | undefined;
+  let sawCoverage = false;
+  let sawSample = false;
+  let sawPrecision = false;
+  const scoreIndexes = new Map(scoreBreakpoints.map((value, index) => [value, index]));
+  const marginIndexes = new Map(marginBreakpoints.map((value, index) => [value, index]));
+  const width = marginBreakpoints.length;
+  const expectedMatchCount = tuneObservations.filter(
+    (observation) => observation.expected_outcome === "matched",
+  ).length;
+  const matchable = tuneObservations.filter(
+    (observation) => observation.expected_outcome !== "no_match",
+  );
+  const retrievalHits = matchable.filter((observation) => {
+    const ids = observation.ranked.slice(0, candidateLimit).map((candidate) => candidate.skill_id);
+    return observation.relevant_skill_ids.some((id) => ids.includes(id));
+  }).length;
+  const retrievalRecall = matchable.length === 0 ? 1 : retrievalHits / matchable.length;
+
+  const evaluateFloor = (floorIndex: number) => {
+    const floor = floorBreakpoints[floorIndex]!;
+    const cells = scoreBreakpoints.length * width;
+    const autoMatches = new Uint32Array(cells);
+    const correctMatches = new Uint32Array(cells);
+    const deliveredDelta = new Int32Array(cells);
+    let ambiguousDeliveredHits = 0;
+
+    for (const observation of tuneObservations) {
+      const top = observation.ranked[0];
+      if (!top || top.score < floor) continue;
+      const second = observation.ranked[1];
+      const margin = second ? top.score - second.score : top.score;
+      const cell = scoreIndexes.get(top.score)! * width + marginIndexes.get(margin)!;
+      autoMatches[cell] = autoMatches[cell]! + 1;
+      const correct = (
+        observation.expected_outcome === "matched" &&
+        top.skill_id === observation.relevant_skill_ids[0]
+      );
+      if (correct) correctMatches[cell] = correctMatches[cell]! + 1;
+
+      if (observation.expected_outcome !== "no_match") {
+        const ambiguousIds = observation.ranked
+          .filter((candidate) => candidate.score >= floor)
+          .slice(0, candidateLimit)
+          .map((candidate) => candidate.skill_id);
+        const ambiguousHit = observation.relevant_skill_ids.some(
+          (id) => ambiguousIds.includes(id),
+        );
+        const matchedHit = observation.relevant_skill_ids.includes(top.skill_id);
+        if (ambiguousHit) ambiguousDeliveredHits++;
+        deliveredDelta[cell] =
+          deliveredDelta[cell]! + Number(matchedHit) - Number(ambiguousHit);
+      }
+    }
 
-  for (const floor of floorBreakpoints) {
-    for (const score of scoreBreakpoints) {
-      if (score < floor) continue;
-      for (const margin of marginBreakpoints) {
-        const candidate: SelectedThresholds = { match_score: score, match_margin: margin, candidate_floor: floor };
-        const m = computeMetrics(tuneObservations, candidate, candidateLimit);
-
-        if (
-          m.auto_match_precision < gates.minAutoMatchPrecision ||
-          m.shortlist_recall_at_5 < gates.minShortlistRecallAt5
-        ) {
-          continue;
+    // A suffix sum turns the score/margin sweep into O(1) metric lookups:
+    // a case auto-matches at every threshold pair at or below its point.
+    for (let scoreIndex = scoreBreakpoints.length - 1; scoreIndex >= 0; scoreIndex--) {
+      for (let marginIndex = width - 1; marginIndex >= 0; marginIndex--) {
+        const cell = scoreIndex * width + marginIndex;
+        if (scoreIndex + 1 < scoreBreakpoints.length) {
+          const below = (scoreIndex + 1) * width + marginIndex;
+          autoMatches[cell] = autoMatches[cell]! + autoMatches[below]!;
+          correctMatches[cell] = correctMatches[cell]! + correctMatches[below]!;
+          deliveredDelta[cell] = deliveredDelta[cell]! + deliveredDelta[below]!;
         }
-
-        if (!best) {
-          best = { thresholds: candidate, metrics: m };
-          continue;
+        if (marginIndex + 1 < width) {
+          const right = cell + 1;
+          autoMatches[cell] = autoMatches[cell]! + autoMatches[right]!;
+          correctMatches[cell] = correctMatches[cell]! + correctMatches[right]!;
+          deliveredDelta[cell] = deliveredDelta[cell]! + deliveredDelta[right]!;
         }
-
-        // Prefer higher coverage, then higher precision, then higher recall, then lower coverage (impossible but symmetry)
-        const bm = best.metrics;
-        if (m.auto_match_coverage > bm.auto_match_coverage) {
-          best = { thresholds: candidate, metrics: m };
-        } else if (m.auto_match_coverage === bm.auto_match_coverage) {
-          if (m.auto_match_precision > bm.auto_match_precision) {
-            best = { thresholds: candidate, metrics: m };
-          } else if (
-            m.auto_match_precision === bm.auto_match_precision &&
-            m.shortlist_recall_at_5 > bm.shortlist_recall_at_5
-          ) {
-            best = { thresholds: candidate, metrics: m };
-          }
+        if (scoreIndex + 1 < scoreBreakpoints.length && marginIndex + 1 < width) {
+          const diagonal = (scoreIndex + 1) * width + marginIndex + 1;
+          autoMatches[cell] = autoMatches[cell]! - autoMatches[diagonal]!;
+          correctMatches[cell] = correctMatches[cell]! - correctMatches[diagonal]!;
+          deliveredDelta[cell] = deliveredDelta[cell]! - deliveredDelta[diagonal]!;
         }
       }
     }
+
+    for (let scoreIndex = 0; scoreIndex < scoreBreakpoints.length; scoreIndex++) {
+      const score = scoreBreakpoints[scoreIndex]!;
+      if (score < floor) continue;
+      for (let marginIndex = 0; marginIndex < marginBreakpoints.length; marginIndex++) {
+        const margin = marginBreakpoints[marginIndex]!;
+        const cell = scoreIndex * width + marginIndex;
+        const autoMatchCount = autoMatches[cell]!;
+        const correctAutoMatchCount = correctMatches[cell]!;
+        const deliveredHits = ambiguousDeliveredHits + deliveredDelta[cell]!;
+        const thresholds = {
+          match_score: score,
+          match_margin: margin,
+          candidate_floor: floor,
+        };
+        const metrics: CalibrationMetrics = {
+          auto_match_precision:
+            autoMatchCount === 0 ? 0 : correctAutoMatchCount / autoMatchCount,
+          auto_match_precision_lower_bound:
+            wilsonLowerBound(correctAutoMatchCount, autoMatchCount),
+          auto_match_coverage:
+            expectedMatchCount === 0 ? 0 : correctAutoMatchCount / expectedMatchCount,
+          auto_match_count: autoMatchCount,
+          correct_auto_match_count: correctAutoMatchCount,
+          retrieval_recall_at_k: retrievalRecall,
+          delivered_shortlist_recall_at_k:
+            matchable.length === 0 ? 1 : deliveredHits / matchable.length,
+        };
+        sawCoverage ||= metrics.auto_match_coverage > 0;
+        sawSample ||= metrics.auto_match_count >= gates.minAutoMatchCount;
+        sawPrecision ||= (
+          metrics.auto_match_precision_lower_bound >= gates.minAutoMatchPrecision
+        );
+        if (!metricsPass(metrics, gates)) continue;
+        const candidate = { thresholds, metrics };
+        if (betterPolicy(candidate, best)) best = candidate;
+      }
+    }
+  };
+
+  // Coarse-to-fine floor search. Floor candidates come from non-top scores, so
+  // this tunes shortlist trimming instead of merely suppressing top matches.
+  const coarse = sampledFloorIndexes(floorBreakpoints.length);
+  for (const index of coarse) evaluateFloor(index);
+  if (best && coarse.length < floorBreakpoints.length) {
+    const bestIndex = floorBreakpoints.indexOf(best.thresholds.candidate_floor);
+    const radius = Math.ceil(floorBreakpoints.length / coarse.length);
+    for (
+      let index = Math.max(0, bestIndex - radius);
+      index <= Math.min(floorBreakpoints.length - 1, bestIndex + radius);
+      index++
+    ) {
+      if (!coarse.includes(index)) evaluateFloor(index);
+    }
   }
 
-  return best?.thresholds;
+  if (best) return { selected: best.thresholds };
+  if (!sawCoverage) return { reason: "no_coverage" };
+  if (!sawSample) return { reason: "insufficient_sample" };
+  if (!sawPrecision) return { reason: "precision_floor_unreachable" };
+  return { reason: "precision_floor_unreachable" };
 }
 
 // ---------------------------------------------------------------------------
@@ -412,12 +751,16 @@ export async function runCalibration(opts: RunCalibrationOptions): Promise<Calib
   const {
     cases,
     getCandidates,
+    getRankedCandidates,
     reranker,
     minAutoMatchPrecision = 0.99,
-    minShortlistRecallAt5 = 0.95,
+    minRetrievalRecallAtK = 0.95,
+    minDeliveredShortlistRecallAtK = minRetrievalRecallAtK,
+    minAutoMatchCount = 30,
+    candidateLimit,
   } = opts;
 
-  if (!reranker) {
+  if (!getRankedCandidates && (!getCandidates || !reranker)) {
     throw new Error(
       "A configured reranker is required to run calibration. " +
         "Configure inference.reranker in your TOML config.",
@@ -427,11 +770,16 @@ export async function runCalibration(opts: RunCalibrationOptions): Promise<Calib
   // --- Step 1: Cache observations (reranker called exactly once per query) ---
   const observations: QueryObservation[] = [];
   for (const c of cases) {
-    const docs = await getCandidates(c.query);
-    const scores = await reranker(c.query, docs);
-    const ranked = docs
-      .map((d, i) => ({ skill_id: d.skill_id, score: scores[i] ?? 0 }))
-      .sort((a, b) => b.score - a.score);
+    let ranked: Array<{ skill_id: string; score: number }>;
+    if (getRankedCandidates) {
+      ranked = await getRankedCandidates(c.query);
+    } else {
+      const docs = await getCandidates!(c.query);
+      const scores = await reranker!(c.query, docs);
+      ranked = docs
+        .map((d, i) => ({ skill_id: d.skill_id, score: scores[i] ?? 0 }))
+        .sort((a, b) => b.score - a.score);
+    }
     observations.push({
       query: c.query,
       split: c.split,
@@ -443,22 +791,61 @@ export async function runCalibration(opts: RunCalibrationOptions): Promise<Calib
 
   // --- Step 2: Select thresholds from tune split only ---
   const tuneObs = observations.filter((o) => o.split === "tune");
-  const selected = selectThresholds(
+  const testObs = observations.filter((o) => o.split === "test");
+  const retrievalThresholds = {
+    match_score: 0,
+    match_margin: 0,
+    candidate_floor: 0,
+  };
+  const tuneRetrieval = computeMetrics(tuneObs, retrievalThresholds, candidateLimit);
+  const testRetrieval = computeMetrics(testObs, retrievalThresholds, candidateLimit);
+  if (
+    tuneRetrieval.retrieval_recall_at_k < minRetrievalRecallAtK ||
+    testRetrieval.retrieval_recall_at_k < minRetrievalRecallAtK
+  ) {
+    return {
+      status: "failed_gates",
+      failed_reason: "recall_precondition_failed",
+      observations,
+    };
+  }
+
+  const gates = {
+    minAutoMatchPrecision,
+    minRetrievalRecallAtK,
+    minDeliveredShortlistRecallAtK,
+    minAutoMatchCount,
+  };
+  const selection = selectThresholds(
     tuneObs,
-    { minAutoMatchPrecision, minShortlistRecallAt5 },
-    5,
+    gates,
+    candidateLimit,
   );
 
-  if (!selected) {
-    return { status: "failed_gates", observations };
+  if (!selection.selected) {
+    return {
+      status: "failed_gates",
+      failed_reason: selection.reason,
+      observations,
+    };
   }
+  const selected = selection.selected;
 
   // --- Step 3: Report tune metrics ---
-  const tune_metrics = computeMetrics(tuneObs, selected);
+  const tune_metrics = computeMetrics(tuneObs, selected, candidateLimit);
 
   // --- Step 4: Evaluate untouched test split ---
-  const testObs = observations.filter((o) => o.split === "test");
-  const test_metrics = computeTestMetrics(testObs, selected);
+  const test_metrics = computeTestMetrics(testObs, selected, candidateLimit);
+  if (!metricsPass(test_metrics, gates)) {
+    return {
+      status: "failed_gates",
+      failed_reason: "test_certification_failed",
+      observations,
+      selected_thresholds: selected,
+      tune_metrics,
+      test_metrics,
+    };
+  }
 
   return { status: "completed", observations, selected_thresholds: selected, tune_metrics, test_metrics };
 }
@@ -479,8 +866,14 @@ export interface CalibrationRunRecord {
   embedding_fingerprint: string;
   corpus_fingerprint: string;
   dataset_hash: string;
+  dataset_provenance?: DatasetProvenanceSummary;
+  candidate_limit: number;
+  attempt_count?: number;
   min_auto_match_precision: number;
+  min_auto_match_count?: number;
+  min_delivered_shortlist_recall_at_k?: number;
   min_shortlist_recall_at_5: number;
+  failed_reason?: CalibrationFailureReason;
   selected_thresholds?: SelectedThresholds;
   tune_metrics?: CalibrationMetrics;
   test_metrics?: CalibrationTestMetrics;
@@ -496,8 +889,15 @@ export interface CalibrationRunSummary {
   embedding_fingerprint: string;
   corpus_fingerprint: string;
   dataset_hash: string;
+  human_labelled_case_count: number;
+  imported_labelled_case_count: number;
+  candidate_limit: number;
+  attempt_count: number;
   min_auto_match_precision: number;
+  min_auto_match_count: number;
+  min_delivered_shortlist_recall_at_k: number;
   min_shortlist_recall_at_5: number;
+  failed_reason?: CalibrationFailureReason;
 }
 
 /**
@@ -525,18 +925,50 @@ export function openCalibrateDb(stateDir: string): Database {
     test_metrics TEXT,
     observations TEXT NOT NULL
   )`);
+  const columns = db.query("PRAGMA table_info(calibration_runs)").all() as Array<{ name: string }>;
+  if (!columns.some((column) => column.name === "candidate_limit")) {
+    db.run("ALTER TABLE calibration_runs ADD COLUMN candidate_limit INTEGER NOT NULL DEFAULT 5");
+  }
+  if (!columns.some((column) => column.name === "attempt_count")) {
+    db.run("ALTER TABLE calibration_runs ADD COLUMN attempt_count INTEGER NOT NULL DEFAULT 1");
+  }
+  if (!columns.some((column) => column.name === "min_auto_match_count")) {
+    db.run("ALTER TABLE calibration_runs ADD COLUMN min_auto_match_count INTEGER NOT NULL DEFAULT 1");
+  }
+  if (!columns.some((column) => column.name === "min_delivered_shortlist_recall_at_k")) {
+    db.run("ALTER TABLE calibration_runs ADD COLUMN min_delivered_shortlist_recall_at_k REAL NOT NULL DEFAULT 0.95");
+  }
+  if (!columns.some((column) => column.name === "failed_reason")) {
+    db.run("ALTER TABLE calibration_runs ADD COLUMN failed_reason TEXT");
+  }
+  if (!columns.some((column) => column.name === "dataset_provenance")) {
+    db.run("ALTER TABLE calibration_runs ADD COLUMN dataset_provenance TEXT NOT NULL DEFAULT '{}'");
+  }
+  if (!columns.some((column) => column.name === "human_labelled_case_count")) {
+    db.run("ALTER TABLE calibration_runs ADD COLUMN human_labelled_case_count INTEGER NOT NULL DEFAULT 0");
+  }
+  if (!columns.some((column) => column.name === "imported_labelled_case_count")) {
+    db.run("ALTER TABLE calibration_runs ADD COLUMN imported_labelled_case_count INTEGER NOT NULL DEFAULT 0");
+  }
   return db;
 }
 
 /** Persist a calibration run (all fields) to the evidence store. */
 export function insertCalibrationRun(db: Database, run: CalibrationRunRecord): void {
+  const attemptCount = (
+    db.query("SELECT COUNT(*) AS count FROM calibration_runs WHERE dataset_hash = ?")
+      .get(run.dataset_hash) as { count: number }
+  ).count + 1;
   db.run(
     `INSERT INTO calibration_runs (
       run_id, created_at, status,
       reranker_fingerprint, embedding_fingerprint, corpus_fingerprint, dataset_hash,
-      min_auto_match_precision, min_shortlist_recall_at_5,
-      selected_thresholds, tune_metrics, test_metrics, observations
-    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      candidate_limit,
+      attempt_count, min_auto_match_precision, min_auto_match_count,
+      min_delivered_shortlist_recall_at_k, min_shortlist_recall_at_5, failed_reason,
+      selected_thresholds, tune_metrics, test_metrics, observations,
+      dataset_provenance, human_labelled_case_count, imported_labelled_case_count
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
     [
       run.run_id,
       run.created_at,
@@ -545,12 +977,20 @@ export function insertCalibrationRun(db: Database, run: CalibrationRunRecord): v
       run.embedding_fingerprint,
       run.corpus_fingerprint,
       run.dataset_hash,
+      run.candidate_limit,
+      attemptCount,
       run.min_auto_match_precision,
+      run.min_auto_match_count ?? 1,
+      run.min_delivered_shortlist_recall_at_k ?? run.min_shortlist_recall_at_5,
       run.min_shortlist_recall_at_5,
+      run.failed_reason ?? null,
       run.selected_thresholds != null ? JSON.stringify(run.selected_thresholds) : null,
       run.tune_metrics != null ? JSON.stringify(run.tune_metrics) : null,
       run.test_metrics != null ? JSON.stringify(run.test_metrics) : null,
       JSON.stringify(run.observations),
+      JSON.stringify(run.dataset_provenance ?? {}),
+      run.dataset_provenance?.human_labelled_case_count ?? 0,
+      run.dataset_provenance?.imported_labelled_case_count ?? 0,
     ],
   );
 }
@@ -563,12 +1003,43 @@ interface RawCalibrationRow {
   embedding_fingerprint: string;
   corpus_fingerprint: string;
   dataset_hash: string;
+  candidate_limit: number;
+  attempt_count: number;
   min_auto_match_precision: number;
+  min_auto_match_count: number;
+  min_delivered_shortlist_recall_at_k: number;
   min_shortlist_recall_at_5: number;
+  failed_reason: string | null;
   selected_thresholds: string | null;
   tune_metrics: string | null;
   test_metrics: string | null;
   observations: string;
+  dataset_provenance: string;
+  human_labelled_case_count: number;
+  imported_labelled_case_count: number;
+}
+
+function parseMetrics(json: string): CalibrationMetrics {
+  const parsed = JSON.parse(json) as Partial<CalibrationMetrics> & {
+    shortlist_recall_at_5?: number;
+    false_no_match_rate?: number;
+  };
+  if (parsed.retrieval_recall_at_k === undefined) {
+    parsed.retrieval_recall_at_k = parsed.shortlist_recall_at_5 ?? 0;
+  }
+  delete parsed.shortlist_recall_at_5;
+  delete parsed.false_no_match_rate;
+  return {
+    auto_match_precision: parsed.auto_match_precision ?? 0,
+    auto_match_precision_lower_bound:
+      parsed.auto_match_precision_lower_bound ?? parsed.auto_match_precision ?? 0,
+    auto_match_coverage: parsed.auto_match_coverage ?? 0,
+    auto_match_count: parsed.auto_match_count ?? 0,
+    correct_auto_match_count: parsed.correct_auto_match_count ?? 0,
+    retrieval_recall_at_k: parsed.retrieval_recall_at_k,
+    delivered_shortlist_recall_at_k:
+      parsed.delivered_shortlist_recall_at_k ?? parsed.retrieval_recall_at_k,
+  };
 }
 
 function rowToRecord(row: RawCalibrationRow): CalibrationRunRecord {
@@ -580,16 +1051,28 @@ function rowToRecord(row: RawCalibrationRow): CalibrationRunRecord {
     embedding_fingerprint: row.embedding_fingerprint,
     corpus_fingerprint: row.corpus_fingerprint,
     dataset_hash: row.dataset_hash,
+    dataset_provenance:
+      Object.keys(JSON.parse(row.dataset_provenance) as object).length > 0
+        ? JSON.parse(row.dataset_provenance) as DatasetProvenanceSummary
+        : undefined,
+    candidate_limit: row.candidate_limit,
+    attempt_count: row.attempt_count,
     min_auto_match_precision: row.min_auto_match_precision,
+    min_auto_match_count: row.min_auto_match_count,
+    min_delivered_shortlist_recall_at_k: row.min_delivered_shortlist_recall_at_k,
     min_shortlist_recall_at_5: row.min_shortlist_recall_at_5,
+    failed_reason: row.failed_reason as CalibrationFailureReason | null ?? undefined,
     selected_thresholds: row.selected_thresholds != null
       ? (JSON.parse(row.selected_thresholds) as SelectedThresholds)
       : undefined,
     tune_metrics: row.tune_metrics != null
-      ? (JSON.parse(row.tune_metrics) as CalibrationMetrics)
+      ? parseMetrics(row.tune_metrics)
       : undefined,
     test_metrics: row.test_metrics != null
-      ? (JSON.parse(row.test_metrics) as CalibrationTestMetrics)
+      ? {
+          ...parseMetrics(row.test_metrics),
+          confusion_matrix: (JSON.parse(row.test_metrics) as CalibrationTestMetrics).confusion_matrix,
+        }
       : undefined,
     observations: JSON.parse(row.observations) as QueryObservation[],
   };
@@ -609,7 +1092,10 @@ export function listCalibrationRuns(db: Database): CalibrationRunSummary[] {
     .query(
       `SELECT run_id, created_at, status,
         reranker_fingerprint, embedding_fingerprint, corpus_fingerprint, dataset_hash,
-        min_auto_match_precision, min_shortlist_recall_at_5
+        candidate_limit,
+        attempt_count, min_auto_match_precision, min_auto_match_count,
+        min_delivered_shortlist_recall_at_k, min_shortlist_recall_at_5, failed_reason,
+        human_labelled_case_count, imported_labelled_case_count
        FROM calibration_runs ORDER BY created_at DESC`,
     )
     .all() as CalibrationRunSummary[];
@@ -685,10 +1171,25 @@ export async function applyCalibrationRun(
       "failed_gates",
     );
   }
+  if (
+    !run.test_metrics ||
+    !metricsPass(run.test_metrics, {
+      minAutoMatchPrecision: run.min_auto_match_precision,
+      minRetrievalRecallAtK: run.min_shortlist_recall_at_5,
+      minDeliveredShortlistRecallAtK:
+        run.min_delivered_shortlist_recall_at_k ?? run.min_shortlist_recall_at_5,
+      minAutoMatchCount: run.min_auto_match_count ?? 1,
+    })
+  ) {
+    throw new ApplyCalibrationError(
+      `Calibration run "${runId}" is not certified on its frozen test split`,
+      "test_certification_failed",
+    );
+  }
 
   // --- Gate 3: fingerprint staleness ---
   if (
-    opts.currentRerankerFingerprint !== undefined &&
+    "currentRerankerFingerprint" in opts &&
     opts.currentRerankerFingerprint !== run.reranker_fingerprint
   ) {
     throw new ApplyCalibrationError(
@@ -740,6 +1241,3 @@ export async function applyCalibrationRun(
     runId,
   });
 }
-
-
-