@klhapp/skillmux 1.0.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/README.md +83 -29
- package/config.remote.example.toml +6 -4
- package/docs/calibration.md +106 -0
- package/docs/configuration.md +51 -7
- package/docs/schema.json +13 -5
- package/package.json +2 -1
- package/src/adapters.ts +111 -38
- package/src/calibrate.ts +623 -125
- package/src/cli.ts +56 -6
- package/src/clients.ts +264 -48
- package/src/config-service.ts +25 -8
- package/src/config-watcher.ts +7 -0
- package/src/config.ts +106 -25
- package/src/dataset-generator.ts +75 -96
- package/src/decision.ts +4 -1
- package/src/doctor.ts +16 -5
- package/src/eval.ts +2 -1
- package/src/router-core.ts +73 -42
- package/src/server.ts +9 -66
- package/src/types.ts +3 -3
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,59 @@ All notable changes to this project are documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [1.2.0](https://github.com/klhq/skillmux/compare/v1.1.0...v1.2.0) (2026-07-28)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
* **calibration:** finish remote contract and docs ([f696da2](https://github.com/klhq/skillmux/commit/f696da25094c19537b1291f694114f8e4052ea69))
|
|
14
|
+
* **calibration:** improve dataset quality ([#88](https://github.com/klhq/skillmux/issues/88)) ([e99b764](https://github.com/klhq/skillmux/commit/e99b764ce90b7432fca0b01c22560ba73b401dd2))
|
|
15
|
+
* **calibration:** require labelled audit feedback ([16d9ec8](https://github.com/klhq/skillmux/commit/16d9ec81acbdca0e42a401b9e28f9e3814b3e0ab))
|
|
16
|
+
* **calibration:** require labelled audit feedback ([#89](https://github.com/klhq/skillmux/issues/89)) ([a7a99e2](https://github.com/klhq/skillmux/commit/a7a99e2d82e619255135b1e941c69c8d6653012c))
|
|
17
|
+
|
|
18
|
+
## [1.1.0](https://github.com/klhq/skillmux/compare/v1.0.1...v1.1.0) (2026-07-28)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
### Added
|
|
22
|
+
|
|
23
|
+
* **calibration:** bind runs to reranker identity ([6ad547a](https://github.com/klhq/skillmux/commit/6ad547a97a6657521fd317d946ab941761fa18b3))
|
|
24
|
+
* **calibration:** certify selected policies ([#86](https://github.com/klhq/skillmux/issues/86)) ([2ced660](https://github.com/klhq/skillmux/commit/2ced660504b7d86dcad94e801944c4e5f2c2d8bd))
|
|
25
|
+
* **calibration:** finish optimizer floor sweep ([#87](https://github.com/klhq/skillmux/issues/87)) ([eed9bc3](https://github.com/klhq/skillmux/commit/eed9bc355f354aa23e6b85d157e9147484457052))
|
|
26
|
+
* **calibration:** honor candidate delivery limit ([#84](https://github.com/klhq/skillmux/issues/84)) ([797d144](https://github.com/klhq/skillmux/commit/797d1443fc0d7b179716f789663220449f5c93d4))
|
|
27
|
+
* **calibration:** make bootstrap reachable ([#85](https://github.com/klhq/skillmux/issues/85)) ([a572d47](https://github.com/klhq/skillmux/commit/a572d47f854e7a2be46504d069d21ccdb346d88c))
|
|
28
|
+
* **config:** reload reranker transport settings ([e15c7af](https://github.com/klhq/skillmux/commit/e15c7afa9dd591f7f838c5747e86d33670afefbf))
|
|
29
|
+
* **config:** require exact embedding endpoints ([3df7584](https://github.com/klhq/skillmux/commit/3df7584749cc3646713d1cb70e6901cc5aab37cc))
|
|
30
|
+
* harden remote embedding clients ([11b2852](https://github.com/klhq/skillmux/commit/11b2852f8c4bf29676da168c01804460570d3061))
|
|
31
|
+
* **inference:** add versioned reranker adapters ([40b85a0](https://github.com/klhq/skillmux/commit/40b85a0b4b9e9877c7eae54477f8a708737a8472))
|
|
32
|
+
* **inference:** add versioned reranker protocol adapters ([#81](https://github.com/klhq/skillmux/issues/81)) ([4e44685](https://github.com/klhq/skillmux/commit/4e446857ed906888034943d4a23877f183596c04))
|
|
33
|
+
* **inference:** harden embedding responses ([f850f8a](https://github.com/klhq/skillmux/commit/f850f8acfd4ab9c78e3b254346b9c2027027a04b))
|
|
34
|
+
|
|
35
|
+
## [Unreleased]
|
|
36
|
+
|
|
37
|
+
### ⚠ BREAKING CHANGES
|
|
38
|
+
|
|
39
|
+
* **inference:** reranker configuration now requires a versioned `adapter` and
|
|
40
|
+
complete `endpoint`; the removed `provider`, `base_url`, and legacy reranker
|
|
41
|
+
base-URL environment variables fail with migration guidance.
|
|
42
|
+
* **inference:** embedding configuration now requires an exact `endpoint`; the
|
|
43
|
+
removed `base_url`, `EMBED_BASE_URL`, `SKILLMUX_EMBED_BASE_URL`, and
|
|
44
|
+
`SKILL_ROUTER_EMBED_BASE_URL` inputs fail with migration guidance.
|
|
45
|
+
|
|
46
|
+
### Added
|
|
47
|
+
|
|
48
|
+
* **inference:** add `jina-v1` and `bifrost-v1` reranker wire-protocol adapters
|
|
49
|
+
with strict indexed-score validation and independent optional Bearer auth.
|
|
50
|
+
* **inference:** validate remote and local embedding vectors before storage and
|
|
51
|
+
classify embedding and reranker failures as configuration, availability, or
|
|
52
|
+
protocol errors.
|
|
53
|
+
|
|
54
|
+
### Changed
|
|
55
|
+
|
|
56
|
+
* **doctor:** incomplete or malformed reranker responses now report degraded or
|
|
57
|
+
unavailable instead of silently filling missing scores with zero.
|
|
58
|
+
* **config:** embedding endpoint, credential-name, and timeout changes reload
|
|
59
|
+
live; model, dimension, device, and dtype changes require restart.
|
|
60
|
+
|
|
8
61
|
## [1.0.1](https://github.com/klhq/skillmux/compare/v1.0.0...v1.0.1) (2026-07-24)
|
|
9
62
|
|
|
10
63
|
|
package/README.md
CHANGED
|
@@ -14,6 +14,7 @@ Built for agents that lack native skill triggering (Goose recipe workers, openco
|
|
|
14
14
|
- [Docker Usage](#docker-usage)
|
|
15
15
|
- [Configuration](#configuration) — inference modes, security scanning, installing skills, env vars
|
|
16
16
|
- [CLI & Automation](docs/cli.md) — context management, remote target resolution, policy calibration, JSON envelopes
|
|
17
|
+
- [Policy Calibration](docs/calibration.md) — labelled datasets, certification, apply lifecycle, and reference profile
|
|
17
18
|
- [Benchmarks & Evaluation](#benchmarks--evaluation)
|
|
18
19
|
- [FAQ & Troubleshooting](#faq--troubleshooting)
|
|
19
20
|
- [Guarantees](#guarantees)
|
|
@@ -37,17 +38,17 @@ resolve_skill("convert this spreadsheet to markdown")
|
|
|
37
38
|
```
|
|
38
39
|
|
|
39
40
|
- **matched** — one skill clearly wins: full `SKILL.md` delivered inline, `sha256(body) == content_sha256 ==` hash of the file on disk at delivery time. Stale index? It re-indexes and delivers fresh bytes — never stale ones.
|
|
40
|
-
- **ambiguous** — up to
|
|
41
|
+
- **ambiguous** — up to 5 candidates (id, title, description). The calling LLM picks and calls `fetch_skill`.
|
|
41
42
|
- **no_match** — proceed under your normal workflow; don't load an unrelated skill.
|
|
42
43
|
|
|
43
44
|
If embeddings are unavailable, the router remains ready with FTS5 lexical retrieval. If an optional reranker is unavailable, it preserves the hybrid shortlist instead of failing.
|
|
44
45
|
|
|
45
46
|
### Tools
|
|
46
47
|
|
|
47
|
-
| Tool
|
|
48
|
-
|
|
49
|
-
| `resolve_skill` | `query`
|
|
50
|
-
| `fetch_skill`
|
|
48
|
+
| Tool | Input | Returns |
|
|
49
|
+
| --------------- | ---------- | ---------------------------------------------------------------------------------------------------------------- |
|
|
50
|
+
| `resolve_skill` | `query` | outcome + metadata in `structuredContent`; on match the verbatim body as text content (exactly once on the wire) |
|
|
51
|
+
| `fetch_skill` | `skill_id` | verbatim body, `content_sha256`, supporting-file paths |
|
|
51
52
|
|
|
52
53
|
The full contract lives in [`docs/schema.json`](docs/schema.json) (JSON Schema 2020-12, language-neutral).
|
|
53
54
|
|
|
@@ -131,12 +132,12 @@ Register with your MCP client directly, e.g.:
|
|
|
131
132
|
|
|
132
133
|
```json
|
|
133
134
|
{
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
135
|
+
"mcpServers": {
|
|
136
|
+
"skillmux": {
|
|
137
|
+
"command": "skillmux",
|
|
138
|
+
"args": ["serve"]
|
|
139
|
+
}
|
|
138
140
|
}
|
|
139
|
-
}
|
|
140
141
|
}
|
|
141
142
|
```
|
|
142
143
|
|
|
@@ -167,9 +168,21 @@ curl -sS -X POST http://127.0.0.1:3000/mcp \
|
|
|
167
168
|
Against the `csv-formatter` skill authored above, that returns a real match — trimmed here for length:
|
|
168
169
|
|
|
169
170
|
```json
|
|
170
|
-
{
|
|
171
|
-
|
|
172
|
-
|
|
171
|
+
{
|
|
172
|
+
"result": {
|
|
173
|
+
"structuredContent": {
|
|
174
|
+
"outcome": "ambiguous",
|
|
175
|
+
"retrieval": "hybrid",
|
|
176
|
+
"candidates": [
|
|
177
|
+
{
|
|
178
|
+
"skill_id": "csv-formatter",
|
|
179
|
+
"title": "CSV Formatter",
|
|
180
|
+
"description": "Converts CSV or spreadsheet data..."
|
|
181
|
+
}
|
|
182
|
+
]
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
173
186
|
```
|
|
174
187
|
|
|
175
188
|
`outcome` is `"ambiguous"` here specifically because the vault only has one skill in it — with more skills installed, a clear top match returns `"matched"` with the full `SKILL.md` body inline instead of a candidate list.
|
|
@@ -184,7 +197,7 @@ bun run src/cli.ts serve
|
|
|
184
197
|
|
|
185
198
|
## Pinning skills across surfaces
|
|
186
199
|
|
|
187
|
-
Optional — skip this if `resolve_skill` alone is enough (most setups). Use it once you want a small set of skills loaded
|
|
200
|
+
Optional — skip this if `resolve_skill` alone is enough (most setups). Use it once you want a small set of skills loaded _statically_ in every agent that reads from a given directory, instead of routed on demand — see [Tiers](#tiers-routed-vs-pinned).
|
|
188
201
|
|
|
189
202
|
Run `skillmux init` with no arguments in a terminal for guided setup. It checks
|
|
190
203
|
the vault, preselects clients with concrete filesystem evidence, asks for core
|
|
@@ -350,7 +363,7 @@ The `skillmux` is packaged and distributed as a Docker image in two variants:
|
|
|
350
363
|
1. **`skillmux:latest`**: Bundles the small quantized GTE embedding model for local hybrid retrieval.
|
|
351
364
|
2. **`skillmux:latest-slim`**: Excludes model weights and supports configured remote embeddings or lexical fallback.
|
|
352
365
|
|
|
353
|
-
Both tags are multi-architecture manifests for Linux AMD64 and ARM64; Docker selects the correct image automatically. Images are published to both [`ghcr.io/klhq/skillmux`](https://github.com/klhq/skillmux/pkgs/container/skillmux) and [`docker.io/
|
|
366
|
+
Both tags are multi-architecture manifests for Linux AMD64 and ARM64; Docker selects the correct image automatically. Images are published to both [`ghcr.io/klhq/skillmux`](https://github.com/klhq/skillmux/pkgs/container/skillmux) and [`docker.io/klhq/skillmux`](https://hub.docker.com/r/klhq/skillmux) — either registry works, examples below use GHCR.
|
|
354
367
|
|
|
355
368
|
### Running HTTP Server (Docker Default)
|
|
356
369
|
|
|
@@ -371,11 +384,12 @@ docker run -d \
|
|
|
371
384
|
-v ~/skills:/vault:ro \
|
|
372
385
|
-v skillmux-data:/data \
|
|
373
386
|
-p 3000:3000 \
|
|
374
|
-
-e
|
|
387
|
+
-e EMBED_ENDPOINT="http://embeddings-host:8080/v1/embeddings" \
|
|
375
388
|
ghcr.io/klhq/skillmux:latest-slim
|
|
376
389
|
```
|
|
377
390
|
|
|
378
391
|
Connect your MCP client to the HTTP endpoint (e.g. standard Streamable HTTP transport):
|
|
392
|
+
|
|
379
393
|
- POST messages to `http://localhost:3000/mcp`
|
|
380
394
|
|
|
381
395
|
#### HTTP server: auth, CORS, rate limiting
|
|
@@ -407,7 +421,7 @@ No config is required for the battery-included local ONNX mode. See [`config.exa
|
|
|
407
421
|
### Inference Modes
|
|
408
422
|
|
|
409
423
|
- The zero-config default combines SQLite FTS5 with the small `Xenova/gte-small` embedding model and returns an ordered shortlist.
|
|
410
|
-
- Configured OpenAI-compatible embeddings replace the local embedder. An optional
|
|
424
|
+
- Configured OpenAI-compatible embeddings replace the local embedder. An optional versioned reranker protocol adapter enables confident automatic matches without coupling configuration to a server product or URL shape.
|
|
411
425
|
|
|
412
426
|
Run `skillmux doctor` to verify routing capability. Run `skillmux config show` to inspect effective configuration; it prints credential variable names, never values.
|
|
413
427
|
|
|
@@ -428,12 +442,12 @@ skillmux scan --fail-on high # exit 1 if any finding is high severity
|
|
|
428
442
|
The v1 rule set covers four categories, each attached to the finding as `rule_id` with a fixed
|
|
429
443
|
`severity`:
|
|
430
444
|
|
|
431
|
-
| `rule_id`
|
|
432
|
-
|
|
433
|
-
| `prompt-injection-phrase` | `high`
|
|
434
|
-
| `invisible-unicode`
|
|
435
|
-
| `secret-pattern`
|
|
436
|
-
| `suspicious-url`
|
|
445
|
+
| `rule_id` | `severity` | Flags |
|
|
446
|
+
| ------------------------- | ---------- | ------------------------------------------------------------------------------------------------- |
|
|
447
|
+
| `prompt-injection-phrase` | `high` | Known instruction-override phrases (e.g. "ignore previous instructions") |
|
|
448
|
+
| `invisible-unicode` | `high` | Zero-width/invisible Unicode code points, including hidden tag-character payloads |
|
|
449
|
+
| `secret-pattern` | `high` | Hardcoded-credential-shaped strings (AWS-style keys, PEM blocks, `api_key=`/`token=` assignments) |
|
|
450
|
+
| `suspicious-url` | `medium` | Bare-IP-address URLs, or URLs paired with exfiltration-suggesting text |
|
|
437
451
|
|
|
438
452
|
`skillmux scan` is unrelated to the `audit` SQLite table / `skillmux report` — that's query telemetry (what got
|
|
439
453
|
routed where); `skillmux scan` is content security (what's in the vault).
|
|
@@ -462,15 +476,18 @@ core/project/routed tier assignment (that's `skillmux sync`'s domain) — it onl
|
|
|
462
476
|
once. Use `skillmux sync` afterward if the installed skill needs to be pinned into a tier.
|
|
463
477
|
|
|
464
478
|
### Environment Variable Overrides
|
|
479
|
+
|
|
465
480
|
All core settings can be overridden via environment variables (handy for Docker):
|
|
481
|
+
|
|
466
482
|
- `VAULT_PATH` — overrides `vault_path` (defaults to `/vault` inside Docker)
|
|
467
483
|
- `STATE_DIR` — overrides `state_dir` (defaults to `/data` inside Docker)
|
|
468
|
-
- `
|
|
484
|
+
- `EMBED_ENDPOINT` / `SKILLMUX_EMBED_ENDPOINT` — overrides the complete remote `inference.embedding.endpoint`
|
|
469
485
|
- `EMBED_MODEL` / `SKILLMUX_EMBED_MODEL` — overrides `embedding.model`
|
|
470
486
|
- `EMBED_DIMENSION` / `SKILLMUX_EMBED_DIMENSION` — overrides `embedding.dimension`
|
|
471
487
|
- `EMBED_DEVICE` / `EMBED_DTYPE` — overrides local `inference.embedding.device` / `inference.embedding.dtype`
|
|
472
|
-
- `
|
|
473
|
-
- `
|
|
488
|
+
- `RERANK_ENDPOINT` / `SKILLMUX_RERANK_ENDPOINT` — overrides the complete remote `inference.reranker.endpoint`
|
|
489
|
+
- `RERANK_ADAPTER` / `SKILLMUX_RERANK_ADAPTER` — overrides `inference.reranker.adapter`
|
|
490
|
+
- `RERANK_MODEL` / `SKILLMUX_RERANK_MODEL` — overrides `inference.reranker.model`
|
|
474
491
|
- `SKILLMUX_CONFIG` — path to custom `config.toml` (default `~/.config/skillmux/config.toml`)
|
|
475
492
|
- `SKILLMUX_MODELS_DIR` — path to directory storing downloaded local models (default `~/.cache/skillmux/models`, `/models` inside Docker)
|
|
476
493
|
- `PORT` — HTTP listen port (default `3000`, HTTP transport only)
|
|
@@ -482,7 +499,40 @@ All core settings can be overridden via environment variables (handy for Docker)
|
|
|
482
499
|
- `HTTP_RATE_LIMIT_RPM` / `SKILLMUX_HTTP_RATE_LIMIT_RPM` — overrides `server.rate_limit.requests_per_minute`
|
|
483
500
|
- `HTTP_RATE_LIMIT_TRUST_PROXY` / `SKILLMUX_HTTP_RATE_LIMIT_TRUST_PROXY` — overrides `server.rate_limit.trust_proxy` (`"true"` to trust `X-Forwarded-For`)
|
|
484
501
|
|
|
485
|
-
Remote API keys are read from the environment variables named by
|
|
502
|
+
Remote API keys are read independently from the environment variables named by
|
|
503
|
+
`inference.embedding.api_key_env` and `inference.reranker.api_key_env`. Omit
|
|
504
|
+
`api_key_env` for an intentionally unauthenticated endpoint. If it is present,
|
|
505
|
+
the named variable must be non-empty before clients are created and is sent as
|
|
506
|
+
a Bearer token. Secret values never live in the config file or diagnostics.
|
|
507
|
+
|
|
508
|
+
Rerankers use an exact endpoint plus an explicit adapter. `jina-v1` sends
|
|
509
|
+
string documents; `bifrost-v1` sends Bifrost document objects. Skillmux never
|
|
510
|
+
infers an adapter from the URL and never adds or removes path components.
|
|
511
|
+
|
|
512
|
+
Embeddings likewise use an exact endpoint and the OpenAI-compatible
|
|
513
|
+
`{ model, input }` contract. Skillmux never adds `/v1/embeddings` or rewrites
|
|
514
|
+
the configured path or query string. Replace the removed `base_url` setting
|
|
515
|
+
with the complete endpoint, for example `base_url = "http://host"` or
|
|
516
|
+
`base_url = "http://host/v1"` becomes
|
|
517
|
+
`endpoint = "http://host/v1/embeddings"`. The removed `EMBED_BASE_URL`,
|
|
518
|
+
`SKILLMUX_EMBED_BASE_URL`, and `SKILL_ROUTER_EMBED_BASE_URL` variables are
|
|
519
|
+
startup errors with migration guidance.
|
|
520
|
+
|
|
521
|
+
> **Breaking reranker migration:** replace `provider = "infinity"` with
|
|
522
|
+
> `adapter = "jina-v1"`, and replace `base_url` with the complete `endpoint`.
|
|
523
|
+
> The old client appended `/rerank`. For example,
|
|
524
|
+
> `base_url = "http://host/v1"` becomes
|
|
525
|
+
> `endpoint = "http://host/v1/rerank"`; a bare `http://host` becomes
|
|
526
|
+
> `http://host/rerank`. The removed `RERANK_BASE_URL`,
|
|
527
|
+
> `SKILLMUX_RERANK_BASE_URL`, and `SKILL_ROUTER_RERANK_BASE_URL` variables are
|
|
528
|
+
> startup errors with migration guidance.
|
|
529
|
+
|
|
530
|
+
Verified reranker contracts:
|
|
531
|
+
|
|
532
|
+
| Implementation | Tested version | Endpoint | Adapter | Verification |
|
|
533
|
+
| ------------------- | -------------: | -------------------- | ------------ | ----------------------------------------------------------------- |
|
|
534
|
+
| Bifrost | 1.6.6 | `/v1/rerank` | `bifrost-v1` | Live end-to-end request and recorded contract fixture, 2026-07-28 |
|
|
535
|
+
| Jina-style contract | fixture | exact configured URL | `jina-v1` | Automated request/response contract suite |
|
|
486
536
|
|
|
487
537
|
## Benchmarks & Evaluation
|
|
488
538
|
|
|
@@ -497,7 +547,11 @@ bun run src/cli.ts eval
|
|
|
497
547
|
# hybrid recall@5: 1.000
|
|
498
548
|
```
|
|
499
549
|
|
|
500
|
-
Custom policy calibration can also be performed against
|
|
550
|
+
Custom policy calibration can also be performed against reviewed,
|
|
551
|
+
domain-specific query datasets using `skillmux calibrate`. See the
|
|
552
|
+
[calibration guide](docs/calibration.md) for the full operator lifecycle and
|
|
553
|
+
the [CLI reference](docs/cli.md#policy-calibration-skillmux-calibrate) for
|
|
554
|
+
command syntax.
|
|
501
555
|
|
|
502
556
|
## FAQ & Troubleshooting
|
|
503
557
|
|
|
@@ -506,7 +560,7 @@ Custom policy calibration can also be performed against domain-specific query lo
|
|
|
506
560
|
|
|
507
561
|
<br>
|
|
508
562
|
|
|
509
|
-
The router returns `"outcome": "ambiguous"` when multiple candidate skills meet retrieval confidence thresholds, or when no single candidate dominates by a sufficient score margin. In this state, up to
|
|
563
|
+
The router returns `"outcome": "ambiguous"` when multiple candidate skills meet retrieval confidence thresholds, or when no single candidate dominates by a sufficient score margin. In this state, up to 5 candidate skill summaries (`skill_id`, `title`, `description`) are returned so the calling LLM can choose the exact skill and invoke `fetch_skill`.
|
|
510
564
|
|
|
511
565
|
</details>
|
|
512
566
|
|
|
@@ -7,18 +7,20 @@ timeout_ms = 5000
|
|
|
7
7
|
|
|
8
8
|
[inference.embedding]
|
|
9
9
|
provider = "openai"
|
|
10
|
-
|
|
10
|
+
endpoint = "https://embedding.example.com/v1/embeddings"
|
|
11
11
|
model = "your-embedding-model"
|
|
12
12
|
dimension = 1024
|
|
13
13
|
api_key_env = "EMBEDDING_API_KEY"
|
|
14
14
|
|
|
15
15
|
[inference.reranker]
|
|
16
|
-
|
|
17
|
-
|
|
16
|
+
adapter = "jina-v1"
|
|
17
|
+
endpoint = "https://reranker.example.com/v1/rerank"
|
|
18
18
|
model = "your-reranker-model"
|
|
19
19
|
api_key_env = "RERANKER_API_KEY"
|
|
20
20
|
|
|
21
|
-
#
|
|
21
|
+
# Optional. Without these values, reranked results remain ambiguous.
|
|
22
|
+
# Calibrate locally against your corpus before enabling automatic matches.
|
|
23
|
+
# See docs/calibration.md; these example values are not universal defaults.
|
|
22
24
|
[inference.thresholds]
|
|
23
25
|
match_score = 0.90
|
|
24
26
|
match_margin = 0.30
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
# Policy calibration
|
|
2
|
+
|
|
3
|
+
Calibration selects the three reranker-score thresholds that turn an ordered
|
|
4
|
+
shortlist into `matched`, `ambiguous`, or `no_match`. It is an operator action,
|
|
5
|
+
not background learning, and it currently runs only against a local Skillmux
|
|
6
|
+
target.
|
|
7
|
+
|
|
8
|
+
## Lifecycle
|
|
9
|
+
|
|
10
|
+
The complete workflow is:
|
|
11
|
+
|
|
12
|
+
```text
|
|
13
|
+
install CLI → configure vault/index/embedding/reranker → obtain labelled dataset
|
|
14
|
+
→ calibrate run → review calibrate show RUN_ID → calibrate apply RUN_ID
|
|
15
|
+
→ live-reloaded policy handles subsequent requests
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
First configure and index the same vault, embedding model, and reranker that
|
|
19
|
+
will serve requests. Supply a reviewed dataset, or generate a starting point
|
|
20
|
+
and review every label:
|
|
21
|
+
|
|
22
|
+
```sh
|
|
23
|
+
skillmux calibrate generate-dataset --out ./eval/queries.json
|
|
24
|
+
skillmux calibrate run --dataset ./eval/queries.json
|
|
25
|
+
skillmux calibrate show RUN_ID
|
|
26
|
+
skillmux calibrate apply RUN_ID
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Skillmux retrieves candidates and reranks exactly once for each evaluation
|
|
30
|
+
query. It caches those observations, searches thresholds on the `tune` split,
|
|
31
|
+
then certifies the selected policy on the frozen `test` split. Calibration
|
|
32
|
+
starts only when an operator invokes `calibrate run`.
|
|
33
|
+
|
|
34
|
+
The operator owns the labels: supply or review the cases, start the run,
|
|
35
|
+
inspect its evidence, and explicitly apply an acceptable result. A successful
|
|
36
|
+
run never changes live thresholds by itself.
|
|
37
|
+
|
|
38
|
+
## Reading a run
|
|
39
|
+
|
|
40
|
+
A `run_id` identifies one immutable calibration attempt and its evidence.
|
|
41
|
+
`calibrate show RUN_ID` is read-only. It reports:
|
|
42
|
+
|
|
43
|
+
- selected thresholds and tune/test metrics;
|
|
44
|
+
- auto-match precision confidence and sample counts;
|
|
45
|
+
- retrieval and delivered-shortlist recall;
|
|
46
|
+
- a closed failure reason when certification fails;
|
|
47
|
+
- reranker, embedding, corpus, and dataset fingerprints;
|
|
48
|
+
- dataset provenance and the number of human-labelled cases; and
|
|
49
|
+
- the attempt count for the dataset hash.
|
|
50
|
+
|
|
51
|
+
`calibrate apply RUN_ID` accepts only a completed, test-certified run. It
|
|
52
|
+
rechecks the reranker fingerprint, rejects thresholds masked by environment
|
|
53
|
+
variables, atomically updates the TOML file, and lets the config watcher
|
|
54
|
+
activate the new snapshot.
|
|
55
|
+
|
|
56
|
+
## Dataset responsibilities
|
|
57
|
+
|
|
58
|
+
Each case needs a query, expected outcome, relevant skill ids, and a fixed
|
|
59
|
+
`tune` or `test` split. Unknown skill ids are rejected. Keep a skill entirely
|
|
60
|
+
within one split so the test set measures generalization rather than memorized
|
|
61
|
+
skill wording.
|
|
62
|
+
|
|
63
|
+
Generated datasets are scaffolding, not ground truth. Review paraphrases,
|
|
64
|
+
near-miss negatives, and ambiguous cases before using them for certification.
|
|
65
|
+
Audit-derived cases require an explicit human label and provenance. Raw audit
|
|
66
|
+
queries are excluded unless the importer is deliberately configured to retain
|
|
67
|
+
them.
|
|
68
|
+
|
|
69
|
+
## When to recalibrate
|
|
70
|
+
|
|
71
|
+
Re-run calibration after a material change to the corpus, embedding or
|
|
72
|
+
retrieval behavior, reranker adapter or model, or after collecting enough new
|
|
73
|
+
human-labelled feedback. Do not recalibrate per user request. Every rerun gets
|
|
74
|
+
a new `run_id`; the active policy remains unchanged until one is applied.
|
|
75
|
+
|
|
76
|
+
## Local and remote targets
|
|
77
|
+
|
|
78
|
+
Calibration is local-only in this release. Local commands operate on the
|
|
79
|
+
configured local vault, index, inference endpoints, dataset path, evidence
|
|
80
|
+
database, and TOML file. Human output always prints `Target: local`; JSON output
|
|
81
|
+
uses `"target": "local"`.
|
|
82
|
+
|
|
83
|
+
Remote servers advertise `"calibration": false`. Every
|
|
84
|
+
`/admin/v1/calibrations` route returns HTTP `501` with
|
|
85
|
+
`error: "not_implemented"`, and the CLI rejects remote calibration before
|
|
86
|
+
uploading or claiming to execute a local dataset path. This also prevents raw
|
|
87
|
+
evaluation queries from being exposed through the admin API.
|
|
88
|
+
|
|
89
|
+
## Reference starting profile
|
|
90
|
+
|
|
91
|
+
Reranker scores are not portable across models, adapters, model revisions, or
|
|
92
|
+
corpora. The profile below is published only to make the checked-in BGE example
|
|
93
|
+
concrete; it is not a certified substitute for calibration.
|
|
94
|
+
|
|
95
|
+
| Model | Adapter | `match_score` | `match_margin` | `candidate_floor` |
|
|
96
|
+
|---|---|---:|---:|---:|
|
|
97
|
+
| `BAAI/bge-reranker-v2-m3` | `jina-v1` | `0.90` | `0.20` | `0.40` |
|
|
98
|
+
|
|
99
|
+
Provenance: the small synthetic corpus and labelled decision cases in
|
|
100
|
+
[`tests/router-core.spec.test.ts`](../tests/router-core.spec.test.ts), with the
|
|
101
|
+
wire contract captured by
|
|
102
|
+
[`tests/fixtures/reranker/jina-v1-request.json`](../tests/fixtures/reranker/jina-v1-request.json).
|
|
103
|
+
That fixture is below the default 30-auto-match certification minimum, so the
|
|
104
|
+
values are a smoke-test/reference profile, not a completed calibration run.
|
|
105
|
+
Run the lifecycle above against the deployment's real corpus before enabling
|
|
106
|
+
automatic matches in production.
|
package/docs/configuration.md
CHANGED
|
@@ -2,7 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
Skillmux defaults to FTS5 plus local GTE-small semantic retrieval. Most users need no config file.
|
|
4
4
|
|
|
5
|
-
For detailed CLI command reference, target resolution,
|
|
5
|
+
For detailed CLI command reference, target resolution, and automation
|
|
6
|
+
envelopes, see [`docs/cli.md`](cli.md). For labelled datasets, threshold
|
|
7
|
+
certification, reference values, and the apply lifecycle, see
|
|
8
|
+
[`docs/calibration.md`](calibration.md).
|
|
6
9
|
|
|
7
10
|
## Machine config bootstrap
|
|
8
11
|
|
|
@@ -51,13 +54,41 @@ Use `skillmux models download` to prefetch models and `skillmux doctor` to verif
|
|
|
51
54
|
|
|
52
55
|
## Remote mode
|
|
53
56
|
|
|
54
|
-
See [`config.remote.example.toml`](../config.remote.example.toml). Embeddings
|
|
57
|
+
See [`config.remote.example.toml`](../config.remote.example.toml). Embeddings
|
|
58
|
+
must implement the OpenAI-compatible `{ model, input }` contract. Configure the
|
|
59
|
+
complete request URL as `inference.embedding.endpoint`; Skillmux does not append
|
|
60
|
+
or rewrite its path or query string. Optional
|
|
61
|
+
reranking uses a versioned wire-protocol adapter and a complete request URL:
|
|
55
62
|
|
|
56
|
-
|
|
63
|
+
```toml
|
|
64
|
+
[inference.reranker]
|
|
65
|
+
adapter = "jina-v1"
|
|
66
|
+
endpoint = "https://reranker.example.com/v1/rerank"
|
|
67
|
+
model = "BAAI/bge-reranker-v2-m3"
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
`jina-v1` sends documents as strings. `bifrost-v1` sends Bifrost document
|
|
71
|
+
objects and requires a Bifrost-style provider-prefixed model name. Skillmux
|
|
72
|
+
does not append `/rerank`, infer the adapter from the URL, or otherwise rewrite
|
|
73
|
+
the endpoint.
|
|
74
|
+
|
|
75
|
+
For embeddings and rerankers independently, omitting `api_key_env` sends no
|
|
76
|
+
`Authorization` header. If `api_key_env` is configured, the named environment
|
|
77
|
+
variable must be non-empty when configuration is loaded; Skillmux sends it as
|
|
78
|
+
a Bearer token. The variable name may appear in diagnostics, but its value
|
|
79
|
+
never does.
|
|
80
|
+
|
|
81
|
+
Remote embedding `dimension` is required. `endpoint`, `api_key_env`, and the
|
|
82
|
+
shared timeout reload live; model, dimension, device, and dtype require a
|
|
83
|
+
restart. Changing only endpoint does not invalidate stored vectors.
|
|
84
|
+
|
|
85
|
+
Reranker adapter and model form the calibration identity. Moving an unchanged
|
|
86
|
+
deployment to another endpoint does not invalidate calibration; changing the
|
|
87
|
+
adapter or model does.
|
|
57
88
|
|
|
58
89
|
## Advanced retrieval
|
|
59
90
|
|
|
60
|
-
Candidate-generation depth
|
|
91
|
+
Candidate-generation depth and agent-context delivery are separate controls:
|
|
61
92
|
|
|
62
93
|
```toml
|
|
63
94
|
[recall]
|
|
@@ -68,9 +99,22 @@ k_vector = 20
|
|
|
68
99
|
candidate_limit = 5
|
|
69
100
|
```
|
|
70
101
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
102
|
+
`recall.k_lexical` and `recall.k_vector` control retrieval depth and cost. The
|
|
103
|
+
reranker receives the fused union from both retrieval lanes. Increasing these
|
|
104
|
+
values can improve retrieval recall, but costs more work.
|
|
105
|
+
|
|
106
|
+
`thresholds.candidate_limit` controls agent context: it caps the ambiguous
|
|
107
|
+
candidate list returned to the calling LLM after retrieval, reranking, and
|
|
108
|
+
threshold filtering. It does not change retrieval depth or the matched,
|
|
109
|
+
ambiguous, or no-match classification.
|
|
110
|
+
|
|
111
|
+
Reranker thresholds have no universal default because score distributions are
|
|
112
|
+
model-specific. Without `inference.thresholds`, Skillmux still uses the
|
|
113
|
+
reranker to order candidates but keeps outcomes ambiguous rather than
|
|
114
|
+
auto-matching. Use `skillmux calibrate run` to select
|
|
115
|
+
`match_score`, `match_margin`, and `candidate_floor`, then explicitly apply the
|
|
116
|
+
certified run. The [calibration guide](calibration.md) also publishes a
|
|
117
|
+
clearly-scoped BGE reference profile for smoke tests.
|
|
74
118
|
|
|
75
119
|
## HTTP server
|
|
76
120
|
|
package/docs/schema.json
CHANGED
|
@@ -224,23 +224,31 @@
|
|
|
224
224
|
"type": "object",
|
|
225
225
|
"properties": {
|
|
226
226
|
"provider": { "const": "openai" },
|
|
227
|
-
"
|
|
227
|
+
"endpoint": {
|
|
228
|
+
"type": "string",
|
|
229
|
+
"format": "uri",
|
|
230
|
+
"description": "Complete HTTP(S) OpenAI-compatible embeddings request URL. Skillmux does not modify its path."
|
|
231
|
+
},
|
|
228
232
|
"model": { "type": "string", "minLength": 1 },
|
|
229
233
|
"dimension": { "type": "integer", "minimum": 1 },
|
|
230
234
|
"api_key_env": { "type": "string", "minLength": 1 }
|
|
231
235
|
},
|
|
232
|
-
"required": ["provider", "
|
|
236
|
+
"required": ["provider", "endpoint", "model", "dimension"],
|
|
233
237
|
"additionalProperties": false
|
|
234
238
|
},
|
|
235
239
|
"reranker": {
|
|
236
240
|
"type": "object",
|
|
237
241
|
"properties": {
|
|
238
|
-
"
|
|
239
|
-
"
|
|
242
|
+
"adapter": { "enum": ["jina-v1", "bifrost-v1"] },
|
|
243
|
+
"endpoint": {
|
|
244
|
+
"type": "string",
|
|
245
|
+
"format": "uri",
|
|
246
|
+
"description": "Complete HTTP(S) rerank request URL. Skillmux does not modify its path."
|
|
247
|
+
},
|
|
240
248
|
"model": { "type": "string", "minLength": 1 },
|
|
241
249
|
"api_key_env": { "type": "string", "minLength": 1 }
|
|
242
250
|
},
|
|
243
|
-
"required": ["
|
|
251
|
+
"required": ["adapter", "endpoint", "model"],
|
|
244
252
|
"additionalProperties": false
|
|
245
253
|
},
|
|
246
254
|
"thresholds": {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@klhapp/skillmux",
|
|
3
|
-
"version": "1.0
|
|
3
|
+
"version": "1.2.0",
|
|
4
4
|
"description": "Local read-only MCP server routing natural-language task queries to skills in a SKILL.md vault, with zero-loss delivery",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"private": false,
|
|
@@ -24,6 +24,7 @@
|
|
|
24
24
|
"src",
|
|
25
25
|
"docs/schema.json",
|
|
26
26
|
"docs/configuration.md",
|
|
27
|
+
"docs/calibration.md",
|
|
27
28
|
"docs/releasing.md",
|
|
28
29
|
"README.md",
|
|
29
30
|
"LICENSE",
|