@klhapp/skillmux 1.5.2 → 1.7.0

package/src/types.ts

@@ -5,11 +5,12 @@ export interface RecallConfig {
 }
 
 export interface OutputConfig {
-  ambiguous_candidate_limit: number;
+  top_k: number;
+  max_top_k: number;
 }
 
 export interface Thresholds {
-  /** @deprecated Use output.ambiguous_candidate_limit instead */
+  /** @deprecated Use output.top_k instead */
   candidate_limit?: number;
   match_score?: number;
   match_margin?: number;
@@ -78,8 +79,6 @@ export interface RemoteInferenceConfig {
   timeout_ms: number;
   embedding: RemoteEmbeddingConfig;
   reranker?: RemoteRerankerConfig;
-  thresholds?: Required<Omit<Thresholds, "candidate_limit">>;
-  calibration?: { run_id: string };
 }
 
 export type InferenceConfig = LocalInferenceConfig | RemoteInferenceConfig;
@@ -114,19 +113,15 @@ export interface Config {
   local_vault_paths: string[];
   state_dir: string;
   recall: RecallConfig;
-  thresholds: Thresholds;
   output: OutputConfig;
   inference: InferenceConfig;
   server?: ServerConfig;
 }
 
-export interface Candidate {
+export interface RankedCandidate {
+  rank: number;
   skill_id: string;
-  title: string;
   description: string;
-}
-
-export interface RankedCandidate extends Candidate {
   score: number | null;
 }
 
@@ -140,40 +135,16 @@ export type DegradationReason =
   | "reranker_unavailable"
   | "reranker_protocol_error";
 
-export interface MatchedResult {
-  outcome: "matched";
-  retrieval: "exact" | "reranked";
-  degraded_from?: "reranked" | "hybrid";
-  degradation_reason?: DegradationReason;
-  skill_id: string;
-  title: string;
-  content_sha256: string;
-  score: number;
-  margin: number;
-  body: string;
-  files: string[];
-}
-
-export interface AmbiguousResult {
-  outcome: "ambiguous";
+export interface ResolveResult {
   retrieval: RetrievalCapability;
   degraded_from?: "reranked" | "hybrid";
   degradation_reason?: DegradationReason;
-  candidates: Candidate[];
+  candidates: RankedCandidate[];
 }
 
-export interface NoMatchResult {
-  outcome: "no_match";
-  retrieval: RetrievalCapability;
-  degraded_from?: "reranked" | "hybrid";
-  degradation_reason?: DegradationReason;
-  message: string;
-}
-
-export type ResolveResult = MatchedResult | AmbiguousResult | NoMatchResult;
-
 export interface ResolveSkillInput {
   query: string;
+  top_k?: number;
   /** Test/ops escape hatch: use lexical retrieval only. Not exposed on the MCP wire. */
   forceLexical?: boolean;
 }
@@ -199,12 +170,10 @@ export interface AuditRow {
   id: number;
   ts: string;
   query: string;
-  outcome: "matched" | "ambiguous" | "no_match";
   retrieval: RetrievalCapability;
   degraded_from?: "reranked" | "hybrid" | null;
   degradation_reason?: DegradationReason | null;
   candidates: AuditCandidate[];
-  selected_skill_id: string | null;
   latency_ms: number;
 }
 

package/docs/calibration.md

@@ -1,162 +0,0 @@
-# Policy calibration
-
-Calibration selects the three reranker-score thresholds that turn an ordered
-shortlist into `matched`, `ambiguous`, or `no_match`. It is an operator action,
-not background learning, and it currently runs only against a local Skillmux
-target.
-
-Read [MCP routing](mcp-routing.md#retrieval-pipeline) before calibrating a new
-retrieval deployment.
-
-## Lifecycle
-
-The complete workflow is:
-
-```text
-install CLI → configure vault/index/embedding/reranker → obtain labelled dataset
-    → calibrate run → review calibrate show RUN_ID → calibrate apply RUN_ID
-    → live-reloaded policy handles subsequent requests
-```
-
-First configure and index the same vault, embedding model, and reranker that
-will serve requests. Supply a reviewed dataset, or generate a starting point
-and review every label:
-
-```sh
-skillmux calibrate generate-dataset --out ./eval/queries.json
-skillmux calibrate run --dataset ./eval/queries.json
-skillmux calibrate show RUN_ID
-skillmux calibrate apply RUN_ID
-```
-
-Skillmux retrieves candidates and reranks exactly once for each evaluation
-query. It runs four queries at a time by default. Set a different positive
-worker limit with `--concurrency N`. The CLI writes completed-case progress to
-stderr without exposing query text.
-
-### Timing report
-
-Add `--timing` to any `calibrate run` invocation to write an aggregate
-performance report to **stderr** after the run finishes (with a completed or
-failed-gates result; a thrown error produces no report). Stdout remains valid
-JSON under `--json --timing`.
-
-```sh
-skillmux calibrate run --dataset ./eval/queries.json --timing
-```
-
-The report uses stable snake_case field names in milliseconds:
-
-| Field | Description |
-|---|---|
-| `cases_total` | Total dataset cases |
-| `cases_executed` | Cases retrieved in this invocation |
-| `cases_reused` | Cases loaded from a prior interrupted run (resume) |
-| `wall_ms` | Wall-clock duration of the full calibrateRun operation |
-| `vault_sync_ms` | One-time vault synchronization before retrieval |
-| `cumulative_embedding_ms` | Total worker time in embedding across all queries |
-| `cumulative_lexical_ms` | Total worker time in lexical search across all queries |
-| `cumulative_vector_ms` | Total worker time in vector search across all queries |
-| `cumulative_reranker_ms` | Total worker time in reranking across all queries |
-| `cumulative_checkpoint_ms` | Total worker time writing observation checkpoints |
-| `policy_evaluation_ms` | Threshold selection and test-split certification |
-
-**Cumulative vs wall time.** The cumulative fields (`cumulative_embedding_ms`,
-`cumulative_lexical_ms`, `cumulative_vector_ms`, `cumulative_reranker_ms`,
-`cumulative_checkpoint_ms`) are total _worker time_ summed across all concurrent
-query retrievals. Because multiple queries run at the same time, the sum of these
-fields typically exceeds `wall_ms`. They measure how much time each stage
-consumed across all workers, not how much wall-clock time each stage accounted
-for. `cases_executed + cases_reused = cases_total`.
-
-Timing collection is fully disabled when `--timing` is absent; it does not affect
-calibration results, resume behavior, checkpoint durability, or JSON schemas.
-Skillmux checkpoints each observation in the calibration evidence database.
-If inference fails or you interrupt the process, find the `running` run with
-`calibrate list` and resume it with the same dataset and certification flags:
-
-```sh
-skillmux calibrate run --dataset ./eval/queries.json --resume RUN_ID
-```
-
-Resume rejects changes to the dataset, corpus, inference models, recall
-settings, candidate limit, or certification gates. After all observations
-exist, Skillmux searches thresholds on the `tune` split and certifies the
-selected policy on the frozen `test` split. Calibration starts only when an
-operator invokes `calibrate run`.
-
-The operator owns the labels: supply or review the cases, start the run,
-inspect its evidence, and explicitly apply an acceptable result. A successful
-run never changes live thresholds by itself.
-
-## Reading a run
-
-A `run_id` identifies one immutable calibration attempt and its evidence.
-`calibrate show RUN_ID` is read-only. It reports:
-
-- selected thresholds and tune/test metrics;
-- auto-match precision confidence and sample counts;
-- retrieval and delivered-shortlist recall;
-- a closed failure reason when certification fails;
-- reranker, embedding, corpus, and dataset fingerprints;
-- dataset provenance and the number of human-labelled cases; and
-- the attempt count for the dataset hash.
-
-`calibrate apply RUN_ID` accepts only a completed, test-certified run. It
-rechecks the reranker fingerprint, rejects thresholds masked by environment
-variables, atomically updates the TOML file, and lets the config watcher
-activate the new snapshot.
-
-## Dataset responsibilities
-
-Each case needs a query, expected outcome, relevant skill ids, and a fixed
-`tune` or `test` split. Unknown skill ids are rejected. Keep a skill entirely
-within one split so the test set measures generalization rather than memorized
-skill wording.
-
-Generated datasets are scaffolding, not ground truth. Review paraphrases,
-near-miss negatives, and ambiguous cases before using them for certification.
-Audit-derived cases require an explicit human label and provenance. Raw audit
-queries are excluded unless the importer is deliberately configured to retain
-them.
-
-## When to recalibrate
-
-Re-run calibration after a material change to the corpus, embedding or
-retrieval behavior, reranker adapter or model, or after collecting enough new
-human-labelled feedback. Do not recalibrate per user request. Every rerun gets
-a new `run_id`; the active policy remains unchanged until one is applied.
-
-## Local and remote targets
-
-Here, `local` and `remote` name CLI administration targets, not inference
-locations or MCP transports. Calibration is local-target-only in this release.
-Local commands operate on the
-configured local vault, index, inference endpoints, dataset path, evidence
-database, and TOML file. Human output always prints `Target: local`; JSON output
-uses `"target": "local"`.
-
-Remote servers advertise `"calibration": false`. Every
-`/admin/v1/calibrations` route returns HTTP `501` with
-`error: "not_implemented"`, and the CLI rejects remote calibration before
-uploading or claiming to execute a local dataset path. This also prevents raw
-evaluation queries from being exposed through the admin API.
-
-## Reference starting profile
-
-Reranker scores are not portable across models, adapters, model revisions, or
-corpora. The profile below is published only to make the checked-in BGE example
-concrete; it is not a certified substitute for calibration.
-
-| Model | Adapter | `match_score` | `match_margin` | `candidate_floor` |
-|---|---|---:|---:|---:|
-| `BAAI/bge-reranker-v2-m3` | `jina-v1` | `0.90` | `0.20` | `0.40` |
-
-Provenance: the small synthetic corpus and labelled decision cases in
-[`tests/router-core.spec.test.ts`](../tests/router-core.spec.test.ts), with the
-wire contract captured by
-[`tests/fixtures/reranker/jina-v1-request.json`](../tests/fixtures/reranker/jina-v1-request.json).
-That fixture is below the default 30-auto-match certification minimum, so the
-values are a smoke-test/reference profile, not a completed calibration run.
-Run the lifecycle above against the deployment's real corpus before enabling
-automatic matches in production.