@klhapp/skillmux 1.6.0 → 1.7.0

package/src/stats.ts

@@ -5,22 +5,32 @@ export const SINCE_PATTERN = /^(\d+[hdwmy]|\d{4}-\d{2}-\d{2}([T ].+)?)$/;
 
 export interface SkillStat {
   skill_id: string;
-  matched_count: number;
   candidate_count: number;
 }
 
-export interface NoMatchQuery {
+export interface EmptyShortlistQuery {
   query: string;
   count: number;
 }
 
+export interface RetrievalTotals {
+  exact: number;
+  reranked: number;
+  hybrid: number;
+  lexical: number;
+}
+
 export interface StatsResponse {
   since: string;
   until: string;
-  outcome_totals: { matched: number; ambiguous: number; no_match: number };
-  ambiguous_rate: number;
+  total_requests: number;
+  empty_shortlist_count: number;
+  empty_shortlist_rate: number;
+  retrieval_totals: RetrievalTotals;
+  degraded_count: number;
+  average_latency_ms: number;
   skills: SkillStat[];
-  top_no_match_queries: NoMatchQuery[];
+  top_empty_shortlist_queries: EmptyShortlistQuery[];
 }
 
 const RELATIVE_WINDOW = /^(\d+)([hdwmy])$/;
@@ -47,58 +57,79 @@ export function parseSince(since: string, now: Date = new Date()): Date {
   return parsed;
 }
 
+function compareCodeUnits(a: string, b: string): number {
+  if (a < b) return -1;
+  if (a > b) return 1;
+  return 0;
+}
+
 export function computeStats(rows: AuditRow[], since: Date, until: Date): StatsResponse {
-  const outcome_totals = { matched: 0, ambiguous: 0, no_match: 0 };
-  const skillStats = new Map<string, { matched_count: number; candidate_count: number }>();
-  const noMatchCounts = new Map<string, number>();
-
-  function statFor(skillId: string) {
-    let stat = skillStats.get(skillId);
-    if (!stat) {
-      stat = { matched_count: 0, candidate_count: 0 };
-      skillStats.set(skillId, stat);
-    }
-    return stat;
-  }
+  const retrieval_totals: RetrievalTotals = { exact: 0, reranked: 0, hybrid: 0, lexical: 0 };
+  const skillCounts = new Map<string, number>();
+  const emptyShortlistCounts = new Map<string, number>();
+  let empty_shortlist_count = 0;
+  let degraded_count = 0;
+  let total_latency_ms = 0;
 
   for (const row of rows) {
-    outcome_totals[row.outcome]++;
-
-    const seenInRow = new Set<string>();
-    for (const candidate of row.candidates) {
-      if (seenInRow.has(candidate.skill_id)) continue;
-      seenInRow.add(candidate.skill_id);
-      statFor(candidate.skill_id).candidate_count++;
+    if (row.retrieval in retrieval_totals) {
+      retrieval_totals[row.retrieval]++;
     }
 
-    if (row.outcome === "matched" && row.selected_skill_id) {
-      statFor(row.selected_skill_id).matched_count++;
+    if (row.degraded_from || row.degradation_reason) {
+      degraded_count++;
     }
 
-    if (row.outcome === "no_match") {
-      noMatchCounts.set(row.query, (noMatchCounts.get(row.query) ?? 0) + 1);
+    total_latency_ms += row.latency_ms;
+
+    if (row.candidates.length === 0) {
+      empty_shortlist_count++;
+      emptyShortlistCounts.set(row.query, (emptyShortlistCounts.get(row.query) ?? 0) + 1);
+    } else {
+      const seenInRow = new Set<string>();
+      for (const candidate of row.candidates) {
+        if (!candidate.skill_id) continue;
+        if (seenInRow.has(candidate.skill_id)) continue;
+        seenInRow.add(candidate.skill_id);
+        skillCounts.set(candidate.skill_id, (skillCounts.get(candidate.skill_id) ?? 0) + 1);
+      }
     }
   }
 
-  const total = outcome_totals.matched + outcome_totals.ambiguous + outcome_totals.no_match;
-  const ambiguous_rate = total > 0 ? outcome_totals.ambiguous / total : 0;
+  const total_requests = rows.length;
+  const empty_shortlist_rate = total_requests > 0 ? empty_shortlist_count / total_requests : 0;
+  const average_latency_ms = total_requests > 0 ? total_latency_ms / total_requests : 0;
 
-  const skills = [...skillStats.entries()]
-    .map(([skill_id, stat]) => ({ skill_id, ...stat }))
-    .sort((a, b) => b.matched_count - a.matched_count);
+  const skills: SkillStat[] = [...skillCounts.entries()]
+    .map(([skill_id, candidate_count]) => ({ skill_id, candidate_count }))
+    .sort((a, b) => {
+      if (b.candidate_count !== a.candidate_count) {
+        return b.candidate_count - a.candidate_count;
+      }
+      return compareCodeUnits(a.skill_id, b.skill_id);
+    });
 
-  const top_no_match_queries = [...noMatchCounts.entries()]
+  const top_empty_shortlist_queries: EmptyShortlistQuery[] = [...emptyShortlistCounts.entries()]
     .map(([query, count]) => ({ query, count }))
-    .sort((a, b) => b.count - a.count)
+    .sort((a, b) => {
+      if (b.count !== a.count) {
+        return b.count - a.count;
+      }
+      return compareCodeUnits(a.query, b.query);
+    })
     .slice(0, 20);
 
   return {
     since: since.toISOString(),
     until: until.toISOString(),
-    outcome_totals,
-    ambiguous_rate,
+    total_requests,
+    empty_shortlist_count,
+    empty_shortlist_rate,
+    retrieval_totals,
+    degraded_count,
+    average_latency_ms,
     skills,
-    top_no_match_queries,
+    top_empty_shortlist_queries,
   };
 }
 
@@ -106,27 +137,60 @@ interface AuditTableRow {
   id: number;
   ts: string;
   query: string;
-  outcome: AuditRow["outcome"];
   retrieval: AuditRow["retrieval"];
+  degraded_from: string | null;
+  degradation_reason: string | null;
   candidates: string;
-  selected_skill_id: string | null;
   latency_ms: number;
 }
 
 export function queryAuditRows(db: Database, sinceIso: string): AuditRow[] {
   const rows = db
-    .query("SELECT id, ts, query, outcome, retrieval, candidates, selected_skill_id, latency_ms FROM audit WHERE ts >= ? ORDER BY ts ASC")
+    .query(
+      "SELECT id, ts, query, retrieval, degraded_from, degradation_reason, candidates, latency_ms FROM audit WHERE ts >= ? ORDER BY ts ASC",
+    )
     .all(sinceIso) as AuditTableRow[];
-  return rows.map((row) => ({
-    id: row.id,
-    ts: row.ts,
-    query: row.query,
-    outcome: row.outcome,
-    retrieval: row.retrieval,
-    candidates: JSON.parse(row.candidates) as AuditCandidate[],
-    selected_skill_id: row.selected_skill_id,
-    latency_ms: row.latency_ms,
-  }));
+
+  return rows.map((row) => {
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(row.candidates);
+    } catch {
+      throw new Error(`Failed to parse candidates JSON for audit row ${row.id}`);
+    }
+    if (!Array.isArray(parsed)) {
+      throw new Error(`Invalid candidates JSON for audit row ${row.id}: expected array, got ${typeof parsed}`);
+    }
+    const candidates: AuditCandidate[] = parsed.map((c: any, index: number) => {
+      if (!c || typeof c !== "object" || Array.isArray(c) || typeof c.skill_id !== "string") {
+        throw new Error(`Invalid candidate at index ${index} for audit row ${row.id}: missing or invalid skill_id`);
+      }
+      const score = c.score;
+      if (score !== null && (typeof score !== "number" || !Number.isFinite(score))) {
+        throw new Error(`Invalid candidate at index ${index} for audit row ${row.id}: missing or invalid score`);
+      }
+      return {
+        skill_id: c.skill_id,
+        score,
+      };
+    });
+
+    const result: AuditRow = {
+      id: row.id,
+      ts: row.ts,
+      query: row.query,
+      retrieval: row.retrieval,
+      candidates,
+      latency_ms: row.latency_ms,
+    };
+    if (row.degraded_from !== null && row.degraded_from !== undefined) {
+      result.degraded_from = row.degraded_from as AuditRow["degraded_from"];
+    }
+    if (row.degradation_reason !== null && row.degradation_reason !== undefined) {
+      result.degradation_reason = row.degradation_reason as AuditRow["degradation_reason"];
+    }
+    return result;
+  });
 }
 
 export function getStats(db: Database, since: string, now: Date = new Date()): StatsResponse {
@@ -139,8 +203,13 @@ export function renderStatsText(stats: StatsResponse): string {
   const lines: string[] = [];
   lines.push(`window: ${stats.since} .. ${stats.until}`);
   lines.push(
-    `outcomes: matched=${stats.outcome_totals.matched} ambiguous=${stats.outcome_totals.ambiguous} ` +
-      `no_match=${stats.outcome_totals.no_match} (ambiguous_rate=${stats.ambiguous_rate.toFixed(3)})`,
+    `requests: total=${stats.total_requests} empty_shortlist=${stats.empty_shortlist_count} ` +
+      `(empty_shortlist_rate=${stats.empty_shortlist_rate.toFixed(3)}) ` +
+      `degraded=${stats.degraded_count} avg_latency_ms=${stats.average_latency_ms.toFixed(1)}`,
+  );
+  lines.push(
+    `retrieval: exact=${stats.retrieval_totals.exact} reranked=${stats.retrieval_totals.reranked} ` +
+      `hybrid=${stats.retrieval_totals.hybrid} lexical=${stats.retrieval_totals.lexical}`,
   );
 
   lines.push("skills:");
@@ -148,15 +217,15 @@ export function renderStatsText(stats: StatsResponse): string {
     lines.push("  (none)");
   } else {
     for (const skill of stats.skills) {
-      lines.push(`  ${skill.skill_id} matched=${skill.matched_count} candidate=${skill.candidate_count}`);
+      lines.push(`  ${skill.skill_id} candidate=${skill.candidate_count}`);
     }
   }
 
-  lines.push("top no_match queries:");
-  if (stats.top_no_match_queries.length === 0) {
+  lines.push("top empty shortlist queries:");
+  if (stats.top_empty_shortlist_queries.length === 0) {
     lines.push("  (none)");
   } else {
-    for (const entry of stats.top_no_match_queries) {
+    for (const entry of stats.top_empty_shortlist_queries) {
       lines.push(`  "${entry.query}" (${entry.count})`);
     }
   }

package/src/types.ts

@@ -5,11 +5,12 @@ export interface RecallConfig {
 }
 
 export interface OutputConfig {
-  ambiguous_candidate_limit: number;
+  top_k: number;
+  max_top_k: number;
 }
 
 export interface Thresholds {
-  /** @deprecated Use output.ambiguous_candidate_limit instead */
+  /** @deprecated Use output.top_k instead */
   candidate_limit?: number;
   match_score?: number;
   match_margin?: number;
@@ -78,8 +79,6 @@ export interface RemoteInferenceConfig {
   timeout_ms: number;
   embedding: RemoteEmbeddingConfig;
   reranker?: RemoteRerankerConfig;
-  thresholds?: Required<Omit<Thresholds, "candidate_limit">>;
-  calibration?: { run_id: string };
 }
 
 export type InferenceConfig = LocalInferenceConfig | RemoteInferenceConfig;
@@ -114,19 +113,15 @@ export interface Config {
   local_vault_paths: string[];
   state_dir: string;
   recall: RecallConfig;
-  thresholds: Thresholds;
   output: OutputConfig;
   inference: InferenceConfig;
   server?: ServerConfig;
 }
 
-export interface Candidate {
+export interface RankedCandidate {
+  rank: number;
   skill_id: string;
-  title: string;
   description: string;
-}
-
-export interface RankedCandidate extends Candidate {
   score: number | null;
 }
 
@@ -140,40 +135,16 @@ export type DegradationReason =
   | "reranker_unavailable"
   | "reranker_protocol_error";
 
-export interface MatchedResult {
-  outcome: "matched";
-  retrieval: "exact" | "reranked";
-  degraded_from?: "reranked" | "hybrid";
-  degradation_reason?: DegradationReason;
-  skill_id: string;
-  title: string;
-  content_sha256: string;
-  score: number;
-  margin: number;
-  body: string;
-  files: string[];
-}
-
-export interface AmbiguousResult {
-  outcome: "ambiguous";
+export interface ResolveResult {
   retrieval: RetrievalCapability;
   degraded_from?: "reranked" | "hybrid";
   degradation_reason?: DegradationReason;
-  candidates: Candidate[];
+  candidates: RankedCandidate[];
 }
 
-export interface NoMatchResult {
-  outcome: "no_match";
-  retrieval: RetrievalCapability;
-  degraded_from?: "reranked" | "hybrid";
-  degradation_reason?: DegradationReason;
-  message: string;
-}
-
-export type ResolveResult = MatchedResult | AmbiguousResult | NoMatchResult;
-
 export interface ResolveSkillInput {
   query: string;
+  top_k?: number;
   /** Test/ops escape hatch: use lexical retrieval only. Not exposed on the MCP wire. */
   forceLexical?: boolean;
 }
@@ -199,12 +170,10 @@ export interface AuditRow {
   id: number;
   ts: string;
   query: string;
-  outcome: "matched" | "ambiguous" | "no_match";
   retrieval: RetrievalCapability;
   degraded_from?: "reranked" | "hybrid" | null;
   degradation_reason?: DegradationReason | null;
   candidates: AuditCandidate[];
-  selected_skill_id: string | null;
   latency_ms: number;
 }
 

package/docs/calibration.md

@@ -1,198 +0,0 @@
-# Policy calibration
-
-Calibration selects the three reranker-score thresholds that turn an ordered
-shortlist into `matched`, `ambiguous`, or `no_match`. It is an operator action,
-not background learning, and it currently runs only against a local Skillmux
-target.
-
-Read [MCP routing](mcp-routing.md#retrieval-pipeline) before calibrating a new
-retrieval deployment.
-
-## Lifecycle
-
-The complete workflow is:
-
-```text
-install CLI → configure vault/index/embedding/reranker → obtain labelled dataset
-    → calibrate run → review calibrate show RUN_ID → calibrate apply RUN_ID
-    → live-reloaded policy handles subsequent requests
-```
-
-First configure and index the same vault, embedding model, and reranker that
-will serve requests. Supply a reviewed dataset, or generate a starting point
-and review every label:
-
-```sh
-skillmux calibrate generate-dataset --out ./eval/queries.json
-skillmux calibrate run --dataset ./eval/queries.json
-skillmux calibrate show RUN_ID
-skillmux calibrate apply RUN_ID
-```
-
-Skillmux retrieves candidates and reranks exactly once for each evaluation
-query. It runs four queries at a time by default. Set a different positive
-worker limit with `--concurrency N`. The CLI writes completed-case progress to
-stderr without exposing query text.
-
-### Timing report
-
-Add `--timing` to any `calibrate run` invocation to write an aggregate
-performance report to **stderr** after the run finishes (with a completed or
-failed-gates result; a thrown error produces no report). Stdout remains valid
-JSON under `--json --timing`.
-
-```sh
-skillmux calibrate run --dataset ./eval/queries.json --timing
-```
-
-The report uses stable snake_case field names in milliseconds:
-
-| Field | Description |
-|---|---|
-| `cases_total` | Total dataset cases |
-| `cases_executed` | Cases retrieved in this invocation |
-| `cases_reused` | Cases loaded from a prior interrupted run (resume) |
-| `wall_ms` | Wall-clock duration of the full calibrateRun operation |
-| `vault_sync_ms` | One-time vault synchronization before retrieval |
-| `cumulative_embedding_ms` | Total worker time in embedding across all queries |
-| `cumulative_lexical_ms` | Total worker time in lexical search across all queries |
-| `cumulative_vector_ms` | Total worker time in vector search across all queries |
-| `cumulative_reranker_ms` | Total worker time in reranking across all queries |
-| `cumulative_checkpoint_ms` | Total worker time writing observation checkpoints |
-| `policy_evaluation_ms` | Threshold selection and test-split certification |
-
-**Cumulative vs wall time.** The cumulative fields (`cumulative_embedding_ms`,
-`cumulative_lexical_ms`, `cumulative_vector_ms`, `cumulative_reranker_ms`,
-`cumulative_checkpoint_ms`) are total _worker time_ summed across all concurrent
-query retrievals. Because multiple queries run at the same time, the sum of these
-fields typically exceeds `wall_ms`. They measure how much time each stage
-consumed across all workers, not how much wall-clock time each stage accounted
-for. `cases_executed + cases_reused = cases_total`.
-
-Timing collection is fully disabled when `--timing` is absent; it does not affect
-calibration results, resume behavior, checkpoint durability, or JSON schemas.
-Skillmux checkpoints each observation in the calibration evidence database.
-If inference fails or you interrupt the process, find the `running` run with
-`calibrate list` and resume it with the same dataset and certification flags:
-
-```sh
-skillmux calibrate run --dataset ./eval/queries.json --resume RUN_ID
-```
-
-Resume rejects changes to the dataset, corpus, inference models, recall
-settings, candidate limit, or certification gates. After all observations
-exist, Skillmux searches thresholds on the `tune` split and certifies the
-selected policy on the frozen `test` split. Calibration starts only when an
-operator invokes `calibrate run`.
-
-The operator owns the labels: supply or review the cases, start the run,
-inspect its evidence, and explicitly apply an acceptable result. A successful
-run never changes live thresholds by itself.
-
-## Certification gates and preflight feasibility
-
-Calibration certifies threshold policies against statistical confidence gates before allowing them to be applied:
-
-| Flag | Default | Description |
-|---|---|---|
-| `--min-auto-match-precision` | `0.75` | Minimum 95% Wilson score lower confidence bound on auto-match precision |
-| `--min-auto-match-count` | `15` | Minimum number of auto-matches required in evaluation |
-| `--min-retrieval-recall-at-k` | `0.95` | Minimum top-k retrieval recall on matchable queries |
-| `--min-delivered-shortlist-recall-at-k` | `0.95` | Minimum delivered shortlist recall on matchable queries |
-| `--tune-auto-match-precision-buffer` | `0.03` | Tune-only selection buffer for Wilson auto-match precision lower bound |
-| `--tune-auto-match-count-buffer` | `3` | Tune-only selection buffer for minimum auto-match count |
-| `--tune-delivered-shortlist-recall-buffer` | `0.02` | Tune-only selection buffer for delivered shortlist recall |
-
-### Tune selection buffers vs production gates
-
-Tune selection buffers ensure that threshold optimization selects policies with sufficient headroom beyond production gates. A candidate policy during tune search must satisfy the production gates plus their respective tune selection buffers. Test-split certification evaluates selected policies against the original production gates without selection buffers.
-
-### Wilson lower confidence bound and evidence size
-
-`min-auto-match-precision` is evaluated not as raw sample accuracy, but as a **95% Wilson score lower confidence bound** ($z \approx 1.960$). This accounts for statistical uncertainty in small datasets.
-
-Because the Wilson lower bound penalizes small sample sizes:
-- A gate of **0.75** lower bound requires at least **15** flawless (15/15) auto-matches ($\text{Wilson}(15, 15) \approx 0.7961$).
-- A 20-case tune matched split can achieve at most $\text{Wilson}(20, 20) \approx 0.8389$.
-- A gate of **0.99** lower bound is statistically impossible on small datasets; it requires at least **381** flawless auto-matches ($\text{Wilson}(381, 381) \approx 0.9900$).
-
-### Preflight feasibility check
-
-To avoid running expensive remote embeddings and rerankings on gates that can never pass, Skillmux executes a **preflight feasibility calculation** immediately after loading the dataset and before creating a running calibration record:
-
-$$\text{effective\_trials} = \max(N_{\text{tune\_matched}}, \text{minAutoMatchCount})$$
-$$\text{max\_attainable\_precision} = \text{WilsonLowerBound}(N_{\text{tune\_matched}}, \text{effective\_trials})$$
-
-If $\text{max\_attainable\_precision} < \text{minAutoMatchPrecision}$, calibration fails immediately with an actionable error indicating the requested precision, requested count, available tune matched cases, and maximum attainable lower bound.
-
-## Reading a run
-
-A `run_id` identifies one immutable calibration attempt and its evidence.
-`calibrate show RUN_ID` is read-only. It reports:
-
-- selected thresholds and tune/test metrics;
-- auto-match precision confidence and sample counts;
-- retrieval and delivered-shortlist recall;
-- a closed failure reason when certification fails;
-- reranker, embedding, corpus, and dataset fingerprints;
-- dataset provenance and the number of human-labelled cases; and
-- the attempt count for the dataset hash.
-
-`calibrate apply RUN_ID` accepts only a completed, test-certified run. It
-rechecks the reranker fingerprint, rejects thresholds masked by environment
-variables, atomically updates the TOML file, and lets the config watcher
-activate the new snapshot.
-
-## Dataset responsibilities
-
-Each case needs a query, expected outcome, relevant skill ids, and a fixed
-`tune` or `test` split. Unknown skill ids are rejected. Keep a skill entirely
-within one split so the test set measures generalization rather than memorized
-skill wording.
-
-Generated datasets are scaffolding, not ground truth. Review paraphrases,
-near-miss negatives, and ambiguous cases before using them for certification.
-Audit-derived cases require an explicit human label and provenance. Raw audit
-queries are excluded unless the importer is deliberately configured to retain
-them.
-
-## When to recalibrate
-
-Re-run calibration after a material change to the corpus, embedding or
-retrieval behavior, reranker adapter or model, or after collecting enough new
-human-labelled feedback. Do not recalibrate per user request. Every rerun gets
-a new `run_id`; the active policy remains unchanged until one is applied.
-
-## Local and remote targets
-
-Here, `local` and `remote` name CLI administration targets, not inference
-locations or MCP transports. Calibration is local-target-only in this release.
-Local commands operate on the
-configured local vault, index, inference endpoints, dataset path, evidence
-database, and TOML file. Human output always prints `Target: local`; JSON output
-uses `"target": "local"`.
-
-Remote servers advertise `"calibration": false`. Every
-`/admin/v1/calibrations` route returns HTTP `501` with
-`error: "not_implemented"`, and the CLI rejects remote calibration before
-uploading or claiming to execute a local dataset path. This also prevents raw
-evaluation queries from being exposed through the admin API.
-
-## Reference starting profile
-
-Reranker scores are not portable across models, adapters, model revisions, or
-corpora. The profile below is published only to make the checked-in BGE example
-concrete; it is not a certified substitute for calibration.
-
-| Model | Adapter | `match_score` | `match_margin` | `candidate_floor` |
-|---|---|---:|---:|---:|
-| `BAAI/bge-reranker-v2-m3` | `jina-v1` | `0.90` | `0.20` | `0.40` |
-
-Provenance: the small synthetic corpus and labelled decision cases in
-[`tests/router-core.spec.test.ts`](../tests/router-core.spec.test.ts), with the
-wire contract captured by
-[`tests/fixtures/reranker/jina-v1-request.json`](../tests/fixtures/reranker/jina-v1-request.json).
-That fixture is below the default 15-auto-match certification minimum, so the
-values are a smoke-test/reference profile, not a completed calibration run.
-Run the lifecycle above against the deployment's real corpus before enabling
-automatic matches in production.