@klhapp/skillmux 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -5,6 +5,58 @@ All notable changes to this project are documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
+ ## [1.1.0](https://github.com/klhq/skillmux/compare/v1.0.1...v1.1.0) (2026-07-28)
9
+
10
+
11
+ ### Added
12
+
13
+ * **calibration:** bind runs to reranker identity ([6ad547a](https://github.com/klhq/skillmux/commit/6ad547a97a6657521fd317d946ab941761fa18b3))
14
+ * **calibration:** certify selected policies ([#86](https://github.com/klhq/skillmux/issues/86)) ([2ced660](https://github.com/klhq/skillmux/commit/2ced660504b7d86dcad94e801944c4e5f2c2d8bd))
15
+ * **calibration:** finish optimizer floor sweep ([#87](https://github.com/klhq/skillmux/issues/87)) ([eed9bc3](https://github.com/klhq/skillmux/commit/eed9bc355f354aa23e6b85d157e9147484457052))
16
+ * **calibration:** honor candidate delivery limit ([#84](https://github.com/klhq/skillmux/issues/84)) ([797d144](https://github.com/klhq/skillmux/commit/797d1443fc0d7b179716f789663220449f5c93d4))
17
+ * **calibration:** make bootstrap reachable ([#85](https://github.com/klhq/skillmux/issues/85)) ([a572d47](https://github.com/klhq/skillmux/commit/a572d47f854e7a2be46504d069d21ccdb346d88c))
18
+ * **config:** reload reranker transport settings ([e15c7af](https://github.com/klhq/skillmux/commit/e15c7afa9dd591f7f838c5747e86d33670afefbf))
19
+ * **config:** require exact embedding endpoints ([3df7584](https://github.com/klhq/skillmux/commit/3df7584749cc3646713d1cb70e6901cc5aab37cc))
20
+ * harden remote embedding clients ([11b2852](https://github.com/klhq/skillmux/commit/11b2852f8c4bf29676da168c01804460570d3061))
21
+ * **inference:** add versioned reranker adapters ([40b85a0](https://github.com/klhq/skillmux/commit/40b85a0b4b9e9877c7eae54477f8a708737a8472))
22
+ * **inference:** add versioned reranker protocol adapters ([#81](https://github.com/klhq/skillmux/issues/81)) ([4e44685](https://github.com/klhq/skillmux/commit/4e446857ed906888034943d4a23877f183596c04))
23
+ * **inference:** harden embedding responses ([f850f8a](https://github.com/klhq/skillmux/commit/f850f8acfd4ab9c78e3b254346b9c2027027a04b))
24
+
25
+ ## [Unreleased]
26
+
27
+ ### ⚠ BREAKING CHANGES
28
+
29
+ * **inference:** reranker configuration now requires a versioned `adapter` and
30
+ complete `endpoint`; the removed `provider`, `base_url`, and legacy reranker
31
+ base-URL environment variables fail with migration guidance.
32
+ * **inference:** embedding configuration now requires an exact `endpoint`; the
33
+ removed `base_url`, `EMBED_BASE_URL`, `SKILLMUX_EMBED_BASE_URL`, and
34
+ `SKILL_ROUTER_EMBED_BASE_URL` inputs fail with migration guidance.
35
+
36
+ ### Added
37
+
38
+ * **inference:** add `jina-v1` and `bifrost-v1` reranker wire-protocol adapters
39
+ with strict indexed-score validation and independent optional Bearer auth.
40
+ * **inference:** validate remote and local embedding vectors before storage and
41
+ classify embedding and reranker failures as configuration, availability, or
42
+ protocol errors.
43
+
44
+ ### Changed
45
+
46
+ * **doctor:** incomplete or malformed reranker responses now report degraded or
47
+ unavailable instead of silently filling missing scores with zero.
48
+ * **config:** embedding endpoint, credential-name, and timeout changes reload
49
+ live; model, dimension, device, and dtype changes require restart.
50
+
51
+ ## [1.0.1](https://github.com/klhq/skillmux/compare/v1.0.0...v1.0.1) (2026-07-24)
52
+
53
+
54
+ ### Changed
55
+
56
+ * **cli:** centralize output envelopes ([#77](https://github.com/klhq/skillmux/issues/77)) ([d457b6b](https://github.com/klhq/skillmux/commit/d457b6b24322efe0eda896ebb76cfd8dc9985819))
57
+ * **cli:** split command modules ([#75](https://github.com/klhq/skillmux/issues/75)) ([b1c2e8b](https://github.com/klhq/skillmux/commit/b1c2e8bf69e0a12c95960729eda581468c9c0646))
58
+ * **config:** extract TOML config-mutation module and share watcher test-utils ([#80](https://github.com/klhq/skillmux/issues/80)) ([d084bd6](https://github.com/klhq/skillmux/commit/d084bd6a56a5985dc92b7362171a0e62e9875926))
59
+
8
60
  ## [1.0.0](https://github.com/klhq/skillmux/compare/v0.6.0...v1.0.0) (2026-07-24)
9
61
 
10
62
 
package/README.md CHANGED
@@ -37,17 +37,17 @@ resolve_skill("convert this spreadsheet to markdown")
37
37
  ```
38
38
 
39
39
  - **matched** — one skill clearly wins: full `SKILL.md` delivered inline, `sha256(body) == content_sha256 ==` hash of the file on disk at delivery time. Stale index? It re-indexes and delivers fresh bytes — never stale ones.
40
- - **ambiguous** — up to 10 candidates (id, title, description). The calling LLM picks and calls `fetch_skill`.
40
+ - **ambiguous** — up to 5 candidates (id, title, description). The calling LLM picks and calls `fetch_skill`.
41
41
  - **no_match** — proceed under your normal workflow; don't load an unrelated skill.
42
42
 
43
43
  If embeddings are unavailable, the router remains ready with FTS5 lexical retrieval. If an optional reranker is unavailable, it preserves the hybrid shortlist instead of failing.
44
44
 
45
45
  ### Tools
46
46
 
47
- | Tool | Input | Returns |
48
- |------|-------|---------|
49
- | `resolve_skill` | `query` | outcome + metadata in `structuredContent`; on match the verbatim body as text content (exactly once on the wire) |
50
- | `fetch_skill` | `skill_id` | verbatim body, `content_sha256`, supporting-file paths |
47
+ | Tool | Input | Returns |
48
+ | --------------- | ---------- | ---------------------------------------------------------------------------------------------------------------- |
49
+ | `resolve_skill` | `query` | outcome + metadata in `structuredContent`; on match the verbatim body as text content (exactly once on the wire) |
50
+ | `fetch_skill` | `skill_id` | verbatim body, `content_sha256`, supporting-file paths |
51
51
 
52
52
  The full contract lives in [`docs/schema.json`](docs/schema.json) (JSON Schema 2020-12, language-neutral).
53
53
 
@@ -131,12 +131,12 @@ Register with your MCP client directly, e.g.:
131
131
 
132
132
  ```json
133
133
  {
134
- "mcpServers": {
135
- "skillmux": {
136
- "command": "skillmux",
137
- "args": ["serve"]
134
+ "mcpServers": {
135
+ "skillmux": {
136
+ "command": "skillmux",
137
+ "args": ["serve"]
138
+ }
138
139
  }
139
- }
140
140
  }
141
141
  ```
142
142
 
@@ -167,9 +167,21 @@ curl -sS -X POST http://127.0.0.1:3000/mcp \
167
167
  Against the `csv-formatter` skill authored above, that returns a real match — trimmed here for length:
168
168
 
169
169
  ```json
170
- {"result":{"structuredContent":{"outcome":"ambiguous","retrieval":"hybrid","candidates":[
171
- {"skill_id":"csv-formatter","title":"CSV Formatter","description":"Converts CSV or spreadsheet data..."}
172
- ]}}}
170
+ {
171
+ "result": {
172
+ "structuredContent": {
173
+ "outcome": "ambiguous",
174
+ "retrieval": "hybrid",
175
+ "candidates": [
176
+ {
177
+ "skill_id": "csv-formatter",
178
+ "title": "CSV Formatter",
179
+ "description": "Converts CSV or spreadsheet data..."
180
+ }
181
+ ]
182
+ }
183
+ }
184
+ }
173
185
  ```
174
186
 
175
187
  `outcome` is `"ambiguous"` here specifically because the vault only has one skill in it — with more skills installed, a clear top match returns `"matched"` with the full `SKILL.md` body inline instead of a candidate list.
@@ -184,7 +196,7 @@ bun run src/cli.ts serve
184
196
 
185
197
  ## Pinning skills across surfaces
186
198
 
187
- Optional — skip this if `resolve_skill` alone is enough (most setups). Use it once you want a small set of skills loaded *statically* in every agent that reads from a given directory, instead of routed on demand — see [Tiers](#tiers-routed-vs-pinned).
199
+ Optional — skip this if `resolve_skill` alone is enough (most setups). Use it once you want a small set of skills loaded _statically_ in every agent that reads from a given directory, instead of routed on demand — see [Tiers](#tiers-routed-vs-pinned).
188
200
 
189
201
  Run `skillmux init` with no arguments in a terminal for guided setup. It checks
190
202
  the vault, preselects clients with concrete filesystem evidence, asks for core
@@ -350,7 +362,7 @@ The `skillmux` is packaged and distributed as a Docker image in two variants:
350
362
  1. **`skillmux:latest`**: Bundles the small quantized GTE embedding model for local hybrid retrieval.
351
363
  2. **`skillmux:latest-slim`**: Excludes model weights and supports configured remote embeddings or lexical fallback.
352
364
 
353
- Both tags are multi-architecture manifests for Linux AMD64 and ARM64; Docker selects the correct image automatically. Images are published to both [`ghcr.io/klhq/skillmux`](https://github.com/klhq/skillmux/pkgs/container/skillmux) and [`docker.io/lazyskyline/skillmux`](https://hub.docker.com/r/lazyskyline/skillmux) — either registry works, examples below use GHCR.
365
+ Both tags are multi-architecture manifests for Linux AMD64 and ARM64; Docker selects the correct image automatically. Images are published to both [`ghcr.io/klhq/skillmux`](https://github.com/klhq/skillmux/pkgs/container/skillmux) and [`docker.io/klhq/skillmux`](https://hub.docker.com/r/klhq/skillmux) — either registry works, examples below use GHCR.
354
366
 
355
367
  ### Running HTTP Server (Docker Default)
356
368
 
@@ -371,11 +383,12 @@ docker run -d \
371
383
  -v ~/skills:/vault:ro \
372
384
  -v skillmux-data:/data \
373
385
  -p 3000:3000 \
374
- -e EMBED_BASE_URL="http://embeddings-host:8080" \
386
+ -e EMBED_ENDPOINT="http://embeddings-host:8080/v1/embeddings" \
375
387
  ghcr.io/klhq/skillmux:latest-slim
376
388
  ```
377
389
 
378
390
  Connect your MCP client to the HTTP endpoint (e.g. standard Streamable HTTP transport):
391
+
379
392
  - POST messages to `http://localhost:3000/mcp`
380
393
 
381
394
  #### HTTP server: auth, CORS, rate limiting
@@ -407,7 +420,7 @@ No config is required for the battery-included local ONNX mode. See [`config.exa
407
420
  ### Inference Modes
408
421
 
409
422
  - The zero-config default combines SQLite FTS5 with the small `Xenova/gte-small` embedding model and returns an ordered shortlist.
410
- - Configured OpenAI-compatible embeddings replace the local embedder. An optional Infinity-compatible reranker enables confident automatic matches.
423
+ - Configured OpenAI-compatible embeddings replace the local embedder. An optional versioned reranker protocol adapter enables confident automatic matches without coupling configuration to a server product or URL shape.
411
424
 
412
425
  Run `skillmux doctor` to verify routing capability. Run `skillmux config show` to inspect effective configuration; it prints credential variable names, never values.
413
426
 
@@ -428,12 +441,12 @@ skillmux scan --fail-on high # exit 1 if any finding is high severity
428
441
  The v1 rule set covers four categories, each attached to the finding as `rule_id` with a fixed
429
442
  `severity`:
430
443
 
431
- | `rule_id` | `severity` | Flags |
432
- |---|---|---|
433
- | `prompt-injection-phrase` | `high` | Known instruction-override phrases (e.g. "ignore previous instructions") |
434
- | `invisible-unicode` | `high` | Zero-width/invisible Unicode code points, including hidden tag-character payloads |
435
- | `secret-pattern` | `high` | Hardcoded-credential-shaped strings (AWS-style keys, PEM blocks, `api_key=`/`token=` assignments) |
436
- | `suspicious-url` | `medium` | Bare-IP-address URLs, or URLs paired with exfiltration-suggesting text |
444
+ | `rule_id` | `severity` | Flags |
445
+ | ------------------------- | ---------- | ------------------------------------------------------------------------------------------------- |
446
+ | `prompt-injection-phrase` | `high` | Known instruction-override phrases (e.g. "ignore previous instructions") |
447
+ | `invisible-unicode` | `high` | Zero-width/invisible Unicode code points, including hidden tag-character payloads |
448
+ | `secret-pattern` | `high` | Hardcoded-credential-shaped strings (AWS-style keys, PEM blocks, `api_key=`/`token=` assignments) |
449
+ | `suspicious-url` | `medium` | Bare-IP-address URLs, or URLs paired with exfiltration-suggesting text |
437
450
 
438
451
  `skillmux scan` is unrelated to the `audit` SQLite table / `skillmux report` — that's query telemetry (what got
439
452
  routed where); `skillmux scan` is content security (what's in the vault).
@@ -462,15 +475,18 @@ core/project/routed tier assignment (that's `skillmux sync`'s domain) — it onl
462
475
  once. Use `skillmux sync` afterward if the installed skill needs to be pinned into a tier.
463
476
 
464
477
  ### Environment Variable Overrides
478
+
465
479
  All core settings can be overridden via environment variables (handy for Docker):
480
+
466
481
  - `VAULT_PATH` — overrides `vault_path` (defaults to `/vault` inside Docker)
467
482
  - `STATE_DIR` — overrides `state_dir` (defaults to `/data` inside Docker)
468
- - `EMBED_BASE_URL` / `SKILLMUX_EMBED_BASE_URL` — overrides remote `inference.embedding.base_url`
483
+ - `EMBED_ENDPOINT` / `SKILLMUX_EMBED_ENDPOINT` — overrides the complete remote `inference.embedding.endpoint`
469
484
  - `EMBED_MODEL` / `SKILLMUX_EMBED_MODEL` — overrides `embedding.model`
470
485
  - `EMBED_DIMENSION` / `SKILLMUX_EMBED_DIMENSION` — overrides `embedding.dimension`
471
486
  - `EMBED_DEVICE` / `EMBED_DTYPE` — overrides local `inference.embedding.device` / `inference.embedding.dtype`
472
- - `RERANK_BASE_URL` / `SKILLMUX_RERANK_BASE_URL` — overrides remote `inference.reranker.base_url`
473
- - `RERANK_MODEL` / `SKILLMUX_RERANK_MODEL` — overrides `rerank.model`
487
+ - `RERANK_ENDPOINT` / `SKILLMUX_RERANK_ENDPOINT` — overrides the complete remote `inference.reranker.endpoint`
488
+ - `RERANK_ADAPTER` / `SKILLMUX_RERANK_ADAPTER` — overrides `inference.reranker.adapter`
489
+ - `RERANK_MODEL` / `SKILLMUX_RERANK_MODEL` — overrides `inference.reranker.model`
474
490
  - `SKILLMUX_CONFIG` — path to custom `config.toml` (default `~/.config/skillmux/config.toml`)
475
491
  - `SKILLMUX_MODELS_DIR` — path to directory storing downloaded local models (default `~/.cache/skillmux/models`, `/models` inside Docker)
476
492
  - `PORT` — HTTP listen port (default `3000`, HTTP transport only)
@@ -482,7 +498,40 @@ All core settings can be overridden via environment variables (handy for Docker)
482
498
  - `HTTP_RATE_LIMIT_RPM` / `SKILLMUX_HTTP_RATE_LIMIT_RPM` — overrides `server.rate_limit.requests_per_minute`
483
499
  - `HTTP_RATE_LIMIT_TRUST_PROXY` / `SKILLMUX_HTTP_RATE_LIMIT_TRUST_PROXY` — overrides `server.rate_limit.trust_proxy` (`"true"` to trust `X-Forwarded-For`)
484
500
 
485
- Remote API keys are read from the environment variables named by `inference.embedding.api_key_env` and `inference.reranker.api_key_env`; no secret ever lives in the config file.
501
+ Remote API keys are read independently from the environment variables named by
502
+ `inference.embedding.api_key_env` and `inference.reranker.api_key_env`. Omit
503
+ `api_key_env` for an intentionally unauthenticated endpoint. If it is present,
504
+ the named variable must be non-empty before clients are created and is sent as
505
+ a Bearer token. Secret values never live in the config file or diagnostics.
506
+
507
+ Rerankers use an exact endpoint plus an explicit adapter. `jina-v1` sends
508
+ string documents; `bifrost-v1` sends Bifrost document objects. Skillmux never
509
+ infers an adapter from the URL and never adds or removes path components.
510
+
511
+ Embeddings likewise use an exact endpoint and the OpenAI-compatible
512
+ `{ model, input }` contract. Skillmux never adds `/v1/embeddings` or rewrites
513
+ the configured path or query string. Replace the removed `base_url` setting
514
+ with the complete endpoint, for example `base_url = "http://host"` or
515
+ `base_url = "http://host/v1"` becomes
516
+ `endpoint = "http://host/v1/embeddings"`. The removed `EMBED_BASE_URL`,
517
+ `SKILLMUX_EMBED_BASE_URL`, and `SKILL_ROUTER_EMBED_BASE_URL` variables are
518
+ startup errors with migration guidance.
519
+
520
+ > **Breaking reranker migration:** replace `provider = "infinity"` with
521
+ > `adapter = "jina-v1"`, and replace `base_url` with the complete `endpoint`.
522
+ > The old client appended `/rerank`. For example,
523
+ > `base_url = "http://host/v1"` becomes
524
+ > `endpoint = "http://host/v1/rerank"`; a bare `http://host` becomes
525
+ > `http://host/rerank`. The removed `RERANK_BASE_URL`,
526
+ > `SKILLMUX_RERANK_BASE_URL`, and `SKILL_ROUTER_RERANK_BASE_URL` variables are
527
+ > startup errors with migration guidance.
528
+
529
+ Verified reranker contracts:
530
+
531
+ | Implementation | Tested version | Endpoint | Adapter | Verification |
532
+ | ------------------- | -------------: | -------------------- | ------------ | ----------------------------------------------------------------- |
533
+ | Bifrost | 1.6.6 | `/v1/rerank` | `bifrost-v1` | Live end-to-end request and recorded contract fixture, 2026-07-28 |
534
+ | Jina-style contract | fixture | exact configured URL | `jina-v1` | Automated request/response contract suite |
486
535
 
487
536
  ## Benchmarks & Evaluation
488
537
 
@@ -506,7 +555,7 @@ Custom policy calibration can also be performed against domain-specific query lo
506
555
 
507
556
  <br>
508
557
 
509
- The router returns `"outcome": "ambiguous"` when multiple candidate skills meet retrieval confidence thresholds, or when no single candidate dominates by a sufficient score margin. In this state, up to 10 candidate skill summaries (`skill_id`, `title`, `description`) are returned so the calling LLM can choose the exact skill and invoke `fetch_skill`.
558
+ The router returns `"outcome": "ambiguous"` when multiple candidate skills meet retrieval confidence thresholds, or when no single candidate dominates by a sufficient score margin. In this state, up to 5 candidate skill summaries (`skill_id`, `title`, `description`) are returned so the calling LLM can choose the exact skill and invoke `fetch_skill`.
510
559
 
511
560
  </details>
512
561
 
@@ -7,14 +7,14 @@ timeout_ms = 5000
7
7
 
8
8
  [inference.embedding]
9
9
  provider = "openai"
10
- base_url = "https://embedding.example.com"
10
+ endpoint = "https://embedding.example.com/v1/embeddings"
11
11
  model = "your-embedding-model"
12
12
  dimension = 1024
13
13
  api_key_env = "EMBEDDING_API_KEY"
14
14
 
15
15
  [inference.reranker]
16
- provider = "infinity"
17
- base_url = "https://reranker.example.com"
16
+ adapter = "jina-v1"
17
+ endpoint = "https://reranker.example.com/v1/rerank"
18
18
  model = "your-reranker-model"
19
19
  api_key_env = "RERANKER_API_KEY"
20
20
 
@@ -51,13 +51,41 @@ Use `skillmux models download` to prefetch models and `skillmux doctor` to verif
51
51
 
52
52
  ## Remote mode
53
53
 
54
- See [`config.remote.example.toml`](../config.remote.example.toml). Embeddings must implement the OpenAI-compatible `POST /v1/embeddings` API. Optional reranking must implement Infinity's `POST /rerank` API. Credentials are read from the environment variables named by `api_key_env`.
54
+ See [`config.remote.example.toml`](../config.remote.example.toml). Embeddings
55
+ must implement the OpenAI-compatible `{ model, input }` contract. Configure the
56
+ complete request URL as `inference.embedding.endpoint`; Skillmux does not append
57
+ or rewrite its path or query string. Optional
58
+ reranking uses a versioned wire-protocol adapter and a complete request URL:
55
59
 
56
- Remote embedding `dimension` is required. Changing the provider, model, or dimension invalidates stored vectors and safely rebuilds them.
60
+ ```toml
61
+ [inference.reranker]
62
+ adapter = "jina-v1"
63
+ endpoint = "https://reranker.example.com/v1/rerank"
64
+ model = "BAAI/bge-reranker-v2-m3"
65
+ ```
66
+
67
+ `jina-v1` sends documents as strings. `bifrost-v1` sends Bifrost document
68
+ objects and requires a Bifrost-style provider-prefixed model name. Skillmux
69
+ does not append `/rerank`, infer the adapter from the URL, or otherwise rewrite
70
+ the endpoint.
71
+
72
+ For embeddings and rerankers independently, omitting `api_key_env` sends no
73
+ `Authorization` header. If `api_key_env` is configured, the named environment
74
+ variable must be non-empty when configuration is loaded; Skillmux sends it as
75
+ a Bearer token. The variable name may appear in diagnostics, but its value
76
+ never does.
77
+
78
+ Remote embedding `dimension` is required. `endpoint`, `api_key_env`, and the
79
+ shared timeout reload live; model, dimension, device, and dtype require a
80
+ restart. Changing only endpoint does not invalidate stored vectors.
81
+
82
+ Reranker adapter and model form the calibration identity. Moving an unchanged
83
+ deployment to another endpoint does not invalidate calibration; changing the
84
+ adapter or model does.
57
85
 
58
86
  ## Advanced retrieval
59
87
 
60
- Candidate-generation depth is configurable but normally does not need tuning:
88
+ Candidate-generation depth and agent-context delivery are separate controls:
61
89
 
62
90
  ```toml
63
91
  [recall]
@@ -68,7 +96,14 @@ k_vector = 20
68
96
  candidate_limit = 5
69
97
  ```
70
98
 
71
- The router considers up to 20 candidates from each retrieval lane but returns at most 5 to the calling LLM.
99
+ `recall.k_lexical` and `recall.k_vector` control retrieval depth and cost. The
100
+ reranker receives the fused union from both retrieval lanes. Increasing these
101
+ values can improve retrieval recall, but costs more work.
102
+
103
+ `thresholds.candidate_limit` controls agent context: it caps the ambiguous
104
+ candidate list returned to the calling LLM after retrieval, reranking, and
105
+ threshold filtering. It does not change retrieval depth or the matched,
106
+ ambiguous, or no-match classification.
72
107
 
73
108
  Reranker thresholds have no universal default because score distributions are model-specific. When configuring a reranker, provide calibrated `inference.thresholds.match_score`, `inference.thresholds.match_margin`, and `inference.thresholds.candidate_floor`; otherwise configuration is rejected rather than silently applying unsuitable values.
74
109
 
package/docs/schema.json CHANGED
@@ -224,23 +224,31 @@
224
224
  "type": "object",
225
225
  "properties": {
226
226
  "provider": { "const": "openai" },
227
- "base_url": { "type": "string", "format": "uri" },
227
+ "endpoint": {
228
+ "type": "string",
229
+ "format": "uri",
230
+ "description": "Complete HTTP(S) OpenAI-compatible embeddings request URL. Skillmux does not modify its path."
231
+ },
228
232
  "model": { "type": "string", "minLength": 1 },
229
233
  "dimension": { "type": "integer", "minimum": 1 },
230
234
  "api_key_env": { "type": "string", "minLength": 1 }
231
235
  },
232
- "required": ["provider", "base_url", "model", "dimension"],
236
+ "required": ["provider", "endpoint", "model", "dimension"],
233
237
  "additionalProperties": false
234
238
  },
235
239
  "reranker": {
236
240
  "type": "object",
237
241
  "properties": {
238
- "provider": { "const": "infinity" },
239
- "base_url": { "type": "string", "format": "uri" },
242
+ "adapter": { "enum": ["jina-v1", "bifrost-v1"] },
243
+ "endpoint": {
244
+ "type": "string",
245
+ "format": "uri",
246
+ "description": "Complete HTTP(S) rerank request URL. Skillmux does not modify its path."
247
+ },
240
248
  "model": { "type": "string", "minLength": 1 },
241
249
  "api_key_env": { "type": "string", "minLength": 1 }
242
250
  },
243
- "required": ["provider", "base_url", "model"],
251
+ "required": ["adapter", "endpoint", "model"],
244
252
  "additionalProperties": false
245
253
  },
246
254
  "thresholds": {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@klhapp/skillmux",
3
- "version": "1.0.0",
3
+ "version": "1.1.0",
4
4
  "description": "Local read-only MCP server routing natural-language task queries to skills in a SKILL.md vault, with zero-loss delivery",
5
5
  "type": "module",
6
6
  "private": false,
package/src/adapters.ts CHANGED
@@ -1,8 +1,9 @@
1
- import { Database } from "bun:sqlite";
1
+ import { createHash } from "node:crypto";
2
2
  import { join } from "node:path";
3
- import { applyCalibrationRun, getCalibrationRun, listCalibrationRuns, loadDecisionCasesFromFile, openCalibrateDb, runCalibration, type CalibrationResult } from "./calibrate";
3
+ import { applyCalibrationRun, getCalibrationRun, insertCalibrationRun, listCalibrationRuns, loadDecisionCasesFromFile, openCalibrateDb, runCalibration, type CalibrationResult } from "./calibrate";
4
4
  import { createClients } from "./clients";
5
- import { DEFAULT_CONFIG_PATH, expandHome, loadConfig } from "./config";
5
+ import { DEFAULT_CONFIG_PATH, embeddingFingerprint, expandHome, loadConfig, rerankerFingerprint } from "./config";
6
+ import { openIndex } from "./db";
6
7
  import { CliError } from "./output";
7
8
  import {
8
9
  computeHash,
@@ -18,7 +19,7 @@ import {
18
19
  type SetConfigResult,
19
20
  } from "./config-service";
20
21
  import type { ResolvedTarget } from "./context";
21
- import { resolveSkill } from "./router-core";
22
+ import { configure, retrieveAndRerank } from "./router-core";
22
23
  import type { Config } from "./types";
23
24
 
24
25
  export interface Capabilities {
@@ -43,7 +44,13 @@ export interface TargetAdapter {
43
44
  configDiff(): Promise<{ diff: Record<string, { prior: unknown; resulting: unknown }> }>;
44
45
  configSet(key: string, rawValStr: string, opts?: { dryRun?: boolean }): Promise<SetConfigResult>;
45
46
  configStatus(): Promise<ConfigStatusResponse>;
46
- calibrateRun(opts?: { datasetPath?: string }): Promise<{ run_id?: string; result?: CalibrationResult }>;
47
+ calibrateRun(opts?: {
48
+ datasetPath?: string;
49
+ minAutoMatchPrecision?: number;
50
+ minRetrievalRecallAtK?: number;
51
+ minDeliveredShortlistRecallAtK?: number;
52
+ minAutoMatchCount?: number;
53
+ }): Promise<{ run_id?: string; result?: CalibrationResult }>;
47
54
  calibrateList(): Promise<any[]>;
48
55
  calibrateShow(runId: string): Promise<any>;
49
56
  calibrateApply(runId: string): Promise<any>;
@@ -123,26 +130,85 @@ export class LocalAdapter implements TargetAdapter {
123
130
  return getLocalConfigStatus(this.configPath);
124
131
  }
125
132
 
126
- async calibrateRun(opts?: { datasetPath?: string }): Promise<{ run_id?: string; result?: CalibrationResult }> {
133
+ async calibrateRun(opts?: {
134
+ datasetPath?: string;
135
+ minAutoMatchPrecision?: number;
136
+ minRetrievalRecallAtK?: number;
137
+ minDeliveredShortlistRecallAtK?: number;
138
+ minAutoMatchCount?: number;
139
+ }): Promise<{ run_id?: string; result?: CalibrationResult }> {
127
140
  const config = await loadConfig(this.configPath);
128
141
  const datasetFile = opts?.datasetPath ?? join(expandHome(config.state_dir), "queries.json");
129
142
  const cases = loadDecisionCasesFromFile(datasetFile);
130
143
  const clients = createClients(config);
144
+ configure({ config, clients });
131
145
  const result = await runCalibration({
132
146
  cases,
133
- getCandidates: async (query: string) => {
134
- const res = await resolveSkill({ query, forceLexical: false });
135
- if (res.outcome === "matched") {
136
- return [{ skill_id: res.skill_id, text: `${res.title} ${res.body}` }];
147
+ getRankedCandidates: async (query: string) => {
148
+ const result = await retrieveAndRerank({ query, forceLexical: false });
149
+ if (result.retrieval !== "reranked") {
150
+ throw new Error(
151
+ "Calibration requires successful hybrid retrieval and reranking for every query.",
152
+ );
137
153
  }
138
- if (res.outcome === "ambiguous") {
139
- return res.candidates.map((c) => ({ skill_id: c.skill_id, text: `${c.title} ${c.description}` }));
140
- }
141
- return [];
154
+ return result.candidates.map((candidate) => ({
155
+ skill_id: candidate.skill_id,
156
+ score: candidate.score ?? 0,
157
+ }));
142
158
  },
143
159
  reranker: clients.rerank,
160
+ candidateLimit: config.thresholds.candidate_limit,
161
+ minAutoMatchPrecision: opts?.minAutoMatchPrecision,
162
+ minRetrievalRecallAtK: opts?.minRetrievalRecallAtK,
163
+ minDeliveredShortlistRecallAtK: opts?.minDeliveredShortlistRecallAtK,
164
+ minAutoMatchCount: opts?.minAutoMatchCount,
144
165
  });
145
- return { result };
166
+ const fingerprint = rerankerFingerprint(config);
167
+ if (!fingerprint) {
168
+ throw new Error("A configured remote reranker is required to record calibration.");
169
+ }
170
+ const datasetText = await Bun.file(datasetFile).text();
171
+ const indexDb = openIndex(expandHome(config.state_dir));
172
+ let corpusFingerprint: string;
173
+ try {
174
+ const rows = indexDb
175
+ .query("SELECT skill_id, content_sha256 FROM skills ORDER BY skill_id")
176
+ .all();
177
+ corpusFingerprint =
178
+ "vault:" +
179
+ createHash("sha256").update(JSON.stringify(rows)).digest("hex");
180
+ } finally {
181
+ indexDb.close();
182
+ }
183
+ const runId = `run_${crypto.randomUUID()}`;
184
+ const db = openCalibrateDb(expandHome(config.state_dir));
185
+ try {
186
+ insertCalibrationRun(db, {
187
+ run_id: runId,
188
+ created_at: new Date().toISOString(),
189
+ status: result.status,
190
+ reranker_fingerprint: fingerprint,
191
+ embedding_fingerprint: embeddingFingerprint(config),
192
+ corpus_fingerprint: corpusFingerprint,
193
+ dataset_hash: createHash("sha256").update(datasetText).digest("hex"),
194
+ candidate_limit: config.thresholds.candidate_limit,
195
+ min_auto_match_precision: opts?.minAutoMatchPrecision ?? 0.99,
196
+ min_auto_match_count: opts?.minAutoMatchCount ?? 30,
197
+ min_delivered_shortlist_recall_at_k:
198
+ opts?.minDeliveredShortlistRecallAtK ??
199
+ opts?.minRetrievalRecallAtK ??
200
+ 0.95,
201
+ min_shortlist_recall_at_5: opts?.minRetrievalRecallAtK ?? 0.95,
202
+ failed_reason: result.failed_reason,
203
+ selected_thresholds: result.selected_thresholds,
204
+ tune_metrics: result.tune_metrics,
205
+ test_metrics: result.test_metrics,
206
+ observations: result.observations,
207
+ });
208
+ } finally {
209
+ db.close();
210
+ }
211
+ return { run_id: runId, result };
146
212
  }
147
213
 
148
214
  async calibrateList(): Promise<any[]> {
@@ -173,7 +239,9 @@ export class LocalAdapter implements TargetAdapter {
173
239
  try {
174
240
  const run = getCalibrationRun(db, runId);
175
241
  if (!run) throw new Error(`Calibration run "${runId}" not found`);
176
- await applyCalibrationRun(db, runId, expandHome(this.configPath), {});
242
+ await applyCalibrationRun(db, runId, expandHome(this.configPath), {
243
+ currentRerankerFingerprint: rerankerFingerprint(config),
244
+ });
177
245
  return { ok: true, run_id: runId };
178
246
  } finally {
179
247
  db.close();
@@ -345,11 +413,23 @@ export class RemoteAdapter implements TargetAdapter {
345
413
  return data.runtime;
346
414
  }
347
415
 
348
- async calibrateRun(opts?: { datasetPath?: string }): Promise<{ run_id?: string; result?: CalibrationResult }> {
416
+ async calibrateRun(opts?: {
417
+ datasetPath?: string;
418
+ minAutoMatchPrecision?: number;
419
+ minRetrievalRecallAtK?: number;
420
+ minDeliveredShortlistRecallAtK?: number;
421
+ minAutoMatchCount?: number;
422
+ }): Promise<{ run_id?: string; result?: CalibrationResult }> {
349
423
  const { status, data } = await this.fetchJson("/admin/v1/calibrations", {
350
424
  method: "POST",
351
425
  headers: { "Content-Type": "application/json" },
352
- body: JSON.stringify({ dataset_path: opts?.datasetPath }),
426
+ body: JSON.stringify({
427
+ dataset_path: opts?.datasetPath,
428
+ min_auto_match_precision: opts?.minAutoMatchPrecision,
429
+ min_retrieval_recall_at_k: opts?.minRetrievalRecallAtK,
430
+ min_delivered_shortlist_recall_at_k: opts?.minDeliveredShortlistRecallAtK,
431
+ min_auto_match_count: opts?.minAutoMatchCount,
432
+ }),
353
433
  });
354
434
  if (status !== 202) {
355
435
  throw new Error(`Remote calibration start failed (${status}): ${data?.message || data}`);