@klhapp/skillmux 1.0.1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/README.md +77 -28
- package/config.remote.example.toml +3 -3
- package/docs/configuration.md +39 -4
- package/docs/schema.json +13 -5
- package/package.json +1 -1
- package/src/adapters.ts +98 -18
- package/src/calibrate.ts +461 -119
- package/src/cli.ts +44 -2
- package/src/clients.ts +264 -48
- package/src/config-service.ts +25 -8
- package/src/config-watcher.ts +7 -0
- package/src/config.ts +106 -25
- package/src/decision.ts +4 -1
- package/src/doctor.ts +16 -5
- package/src/eval.ts +2 -1
- package/src/router-core.ts +73 -42
- package/src/server.ts +8 -5
- package/src/types.ts +3 -3
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,49 @@ All notable changes to this project are documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [1.1.0](https://github.com/klhq/skillmux/compare/v1.0.1...v1.1.0) (2026-07-28)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
* **calibration:** bind runs to reranker identity ([6ad547a](https://github.com/klhq/skillmux/commit/6ad547a97a6657521fd317d946ab941761fa18b3))
|
|
14
|
+
* **calibration:** certify selected policies ([#86](https://github.com/klhq/skillmux/issues/86)) ([2ced660](https://github.com/klhq/skillmux/commit/2ced660504b7d86dcad94e801944c4e5f2c2d8bd))
|
|
15
|
+
* **calibration:** finish optimizer floor sweep ([#87](https://github.com/klhq/skillmux/issues/87)) ([eed9bc3](https://github.com/klhq/skillmux/commit/eed9bc355f354aa23e6b85d157e9147484457052))
|
|
16
|
+
* **calibration:** honor candidate delivery limit ([#84](https://github.com/klhq/skillmux/issues/84)) ([797d144](https://github.com/klhq/skillmux/commit/797d1443fc0d7b179716f789663220449f5c93d4))
|
|
17
|
+
* **calibration:** make bootstrap reachable ([#85](https://github.com/klhq/skillmux/issues/85)) ([a572d47](https://github.com/klhq/skillmux/commit/a572d47f854e7a2be46504d069d21ccdb346d88c))
|
|
18
|
+
* **config:** reload reranker transport settings ([e15c7af](https://github.com/klhq/skillmux/commit/e15c7afa9dd591f7f838c5747e86d33670afefbf))
|
|
19
|
+
* **config:** require exact embedding endpoints ([3df7584](https://github.com/klhq/skillmux/commit/3df7584749cc3646713d1cb70e6901cc5aab37cc))
|
|
20
|
+
* harden remote embedding clients ([11b2852](https://github.com/klhq/skillmux/commit/11b2852f8c4bf29676da168c01804460570d3061))
|
|
21
|
+
* **inference:** add versioned reranker adapters ([40b85a0](https://github.com/klhq/skillmux/commit/40b85a0b4b9e9877c7eae54477f8a708737a8472))
|
|
22
|
+
* **inference:** add versioned reranker protocol adapters ([#81](https://github.com/klhq/skillmux/issues/81)) ([4e44685](https://github.com/klhq/skillmux/commit/4e446857ed906888034943d4a23877f183596c04))
|
|
23
|
+
* **inference:** harden embedding responses ([f850f8a](https://github.com/klhq/skillmux/commit/f850f8acfd4ab9c78e3b254346b9c2027027a04b))
|
|
24
|
+
|
|
25
|
+
## [Unreleased]
|
|
26
|
+
|
|
27
|
+
### ⚠ BREAKING CHANGES
|
|
28
|
+
|
|
29
|
+
* **inference:** reranker configuration now requires a versioned `adapter` and
|
|
30
|
+
complete `endpoint`; the removed `provider`, `base_url`, and legacy reranker
|
|
31
|
+
base-URL environment variables fail with migration guidance.
|
|
32
|
+
* **inference:** embedding configuration now requires an exact `endpoint`; the
|
|
33
|
+
removed `base_url`, `EMBED_BASE_URL`, `SKILLMUX_EMBED_BASE_URL`, and
|
|
34
|
+
`SKILL_ROUTER_EMBED_BASE_URL` inputs fail with migration guidance.
|
|
35
|
+
|
|
36
|
+
### Added
|
|
37
|
+
|
|
38
|
+
* **inference:** add `jina-v1` and `bifrost-v1` reranker wire-protocol adapters
|
|
39
|
+
with strict indexed-score validation and independent optional Bearer auth.
|
|
40
|
+
* **inference:** validate remote and local embedding vectors before storage and
|
|
41
|
+
classify embedding and reranker failures as configuration, availability, or
|
|
42
|
+
protocol errors.
|
|
43
|
+
|
|
44
|
+
### Changed
|
|
45
|
+
|
|
46
|
+
* **doctor:** incomplete or malformed reranker responses now report degraded or
|
|
47
|
+
unavailable instead of silently filling missing scores with zero.
|
|
48
|
+
* **config:** embedding endpoint, credential-name, and timeout changes reload
|
|
49
|
+
live; model, dimension, device, and dtype changes require restart.
|
|
50
|
+
|
|
8
51
|
## [1.0.1](https://github.com/klhq/skillmux/compare/v1.0.0...v1.0.1) (2026-07-24)
|
|
9
52
|
|
|
10
53
|
|
package/README.md
CHANGED
|
@@ -37,17 +37,17 @@ resolve_skill("convert this spreadsheet to markdown")
|
|
|
37
37
|
```
|
|
38
38
|
|
|
39
39
|
- **matched** — one skill clearly wins: full `SKILL.md` delivered inline, `sha256(body) == content_sha256 ==` hash of the file on disk at delivery time. Stale index? It re-indexes and delivers fresh bytes — never stale ones.
|
|
40
|
-
- **ambiguous** — up to
|
|
40
|
+
- **ambiguous** — up to 5 candidates (id, title, description). The calling LLM picks and calls `fetch_skill`.
|
|
41
41
|
- **no_match** — proceed under your normal workflow; don't load an unrelated skill.
|
|
42
42
|
|
|
43
43
|
If embeddings are unavailable, the router remains ready with FTS5 lexical retrieval. If an optional reranker is unavailable, it preserves the hybrid shortlist instead of failing.
|
|
44
44
|
|
|
45
45
|
### Tools
|
|
46
46
|
|
|
47
|
-
| Tool
|
|
48
|
-
|
|
49
|
-
| `resolve_skill` | `query`
|
|
50
|
-
| `fetch_skill`
|
|
47
|
+
| Tool | Input | Returns |
|
|
48
|
+
| --------------- | ---------- | ---------------------------------------------------------------------------------------------------------------- |
|
|
49
|
+
| `resolve_skill` | `query` | outcome + metadata in `structuredContent`; on match the verbatim body as text content (exactly once on the wire) |
|
|
50
|
+
| `fetch_skill` | `skill_id` | verbatim body, `content_sha256`, supporting-file paths |
|
|
51
51
|
|
|
52
52
|
The full contract lives in [`docs/schema.json`](docs/schema.json) (JSON Schema 2020-12, language-neutral).
|
|
53
53
|
|
|
@@ -131,12 +131,12 @@ Register with your MCP client directly, e.g.:
|
|
|
131
131
|
|
|
132
132
|
```json
|
|
133
133
|
{
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
134
|
+
"mcpServers": {
|
|
135
|
+
"skillmux": {
|
|
136
|
+
"command": "skillmux",
|
|
137
|
+
"args": ["serve"]
|
|
138
|
+
}
|
|
138
139
|
}
|
|
139
|
-
}
|
|
140
140
|
}
|
|
141
141
|
```
|
|
142
142
|
|
|
@@ -167,9 +167,21 @@ curl -sS -X POST http://127.0.0.1:3000/mcp \
|
|
|
167
167
|
Against the `csv-formatter` skill authored above, that returns a real match — trimmed here for length:
|
|
168
168
|
|
|
169
169
|
```json
|
|
170
|
-
{
|
|
171
|
-
|
|
172
|
-
|
|
170
|
+
{
|
|
171
|
+
"result": {
|
|
172
|
+
"structuredContent": {
|
|
173
|
+
"outcome": "ambiguous",
|
|
174
|
+
"retrieval": "hybrid",
|
|
175
|
+
"candidates": [
|
|
176
|
+
{
|
|
177
|
+
"skill_id": "csv-formatter",
|
|
178
|
+
"title": "CSV Formatter",
|
|
179
|
+
"description": "Converts CSV or spreadsheet data..."
|
|
180
|
+
}
|
|
181
|
+
]
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
173
185
|
```
|
|
174
186
|
|
|
175
187
|
`outcome` is `"ambiguous"` here specifically because the vault only has one skill in it — with more skills installed, a clear top match returns `"matched"` with the full `SKILL.md` body inline instead of a candidate list.
|
|
@@ -184,7 +196,7 @@ bun run src/cli.ts serve
|
|
|
184
196
|
|
|
185
197
|
## Pinning skills across surfaces
|
|
186
198
|
|
|
187
|
-
Optional — skip this if `resolve_skill` alone is enough (most setups). Use it once you want a small set of skills loaded
|
|
199
|
+
Optional — skip this if `resolve_skill` alone is enough (most setups). Use it once you want a small set of skills loaded _statically_ in every agent that reads from a given directory, instead of routed on demand — see [Tiers](#tiers-routed-vs-pinned).
|
|
188
200
|
|
|
189
201
|
Run `skillmux init` with no arguments in a terminal for guided setup. It checks
|
|
190
202
|
the vault, preselects clients with concrete filesystem evidence, asks for core
|
|
@@ -350,7 +362,7 @@ The `skillmux` is packaged and distributed as a Docker image in two variants:
|
|
|
350
362
|
1. **`skillmux:latest`**: Bundles the small quantized GTE embedding model for local hybrid retrieval.
|
|
351
363
|
2. **`skillmux:latest-slim`**: Excludes model weights and supports configured remote embeddings or lexical fallback.
|
|
352
364
|
|
|
353
|
-
Both tags are multi-architecture manifests for Linux AMD64 and ARM64; Docker selects the correct image automatically. Images are published to both [`ghcr.io/klhq/skillmux`](https://github.com/klhq/skillmux/pkgs/container/skillmux) and [`docker.io/
|
|
365
|
+
Both tags are multi-architecture manifests for Linux AMD64 and ARM64; Docker selects the correct image automatically. Images are published to both [`ghcr.io/klhq/skillmux`](https://github.com/klhq/skillmux/pkgs/container/skillmux) and [`docker.io/klhq/skillmux`](https://hub.docker.com/r/klhq/skillmux) — either registry works, examples below use GHCR.
|
|
354
366
|
|
|
355
367
|
### Running HTTP Server (Docker Default)
|
|
356
368
|
|
|
@@ -371,11 +383,12 @@ docker run -d \
|
|
|
371
383
|
-v ~/skills:/vault:ro \
|
|
372
384
|
-v skillmux-data:/data \
|
|
373
385
|
-p 3000:3000 \
|
|
374
|
-
-e
|
|
386
|
+
-e EMBED_ENDPOINT="http://embeddings-host:8080/v1/embeddings" \
|
|
375
387
|
ghcr.io/klhq/skillmux:latest-slim
|
|
376
388
|
```
|
|
377
389
|
|
|
378
390
|
Connect your MCP client to the HTTP endpoint (e.g. standard Streamable HTTP transport):
|
|
391
|
+
|
|
379
392
|
- POST messages to `http://localhost:3000/mcp`
|
|
380
393
|
|
|
381
394
|
#### HTTP server: auth, CORS, rate limiting
|
|
@@ -407,7 +420,7 @@ No config is required for the battery-included local ONNX mode. See [`config.exa
|
|
|
407
420
|
### Inference Modes
|
|
408
421
|
|
|
409
422
|
- The zero-config default combines SQLite FTS5 with the small `Xenova/gte-small` embedding model and returns an ordered shortlist.
|
|
410
|
-
- Configured OpenAI-compatible embeddings replace the local embedder. An optional
|
|
423
|
+
- Configured OpenAI-compatible embeddings replace the local embedder. An optional versioned reranker protocol adapter enables confident automatic matches without coupling configuration to a server product or URL shape.
|
|
411
424
|
|
|
412
425
|
Run `skillmux doctor` to verify routing capability. Run `skillmux config show` to inspect effective configuration; it prints credential variable names, never values.
|
|
413
426
|
|
|
@@ -428,12 +441,12 @@ skillmux scan --fail-on high # exit 1 if any finding is high severity
|
|
|
428
441
|
The v1 rule set covers four categories, each attached to the finding as `rule_id` with a fixed
|
|
429
442
|
`severity`:
|
|
430
443
|
|
|
431
|
-
| `rule_id`
|
|
432
|
-
|
|
433
|
-
| `prompt-injection-phrase` | `high`
|
|
434
|
-
| `invisible-unicode`
|
|
435
|
-
| `secret-pattern`
|
|
436
|
-
| `suspicious-url`
|
|
444
|
+
| `rule_id` | `severity` | Flags |
|
|
445
|
+
| ------------------------- | ---------- | ------------------------------------------------------------------------------------------------- |
|
|
446
|
+
| `prompt-injection-phrase` | `high` | Known instruction-override phrases (e.g. "ignore previous instructions") |
|
|
447
|
+
| `invisible-unicode` | `high` | Zero-width/invisible Unicode code points, including hidden tag-character payloads |
|
|
448
|
+
| `secret-pattern` | `high` | Hardcoded-credential-shaped strings (AWS-style keys, PEM blocks, `api_key=`/`token=` assignments) |
|
|
449
|
+
| `suspicious-url` | `medium` | Bare-IP-address URLs, or URLs paired with exfiltration-suggesting text |
|
|
437
450
|
|
|
438
451
|
`skillmux scan` is unrelated to the `audit` SQLite table / `skillmux report` — that's query telemetry (what got
|
|
439
452
|
routed where); `skillmux scan` is content security (what's in the vault).
|
|
@@ -462,15 +475,18 @@ core/project/routed tier assignment (that's `skillmux sync`'s domain) — it onl
|
|
|
462
475
|
once. Use `skillmux sync` afterward if the installed skill needs to be pinned into a tier.
|
|
463
476
|
|
|
464
477
|
### Environment Variable Overrides
|
|
478
|
+
|
|
465
479
|
All core settings can be overridden via environment variables (handy for Docker):
|
|
480
|
+
|
|
466
481
|
- `VAULT_PATH` — overrides `vault_path` (defaults to `/vault` inside Docker)
|
|
467
482
|
- `STATE_DIR` — overrides `state_dir` (defaults to `/data` inside Docker)
|
|
468
|
-
- `
|
|
483
|
+
- `EMBED_ENDPOINT` / `SKILLMUX_EMBED_ENDPOINT` — overrides the complete remote `inference.embedding.endpoint`
|
|
469
484
|
- `EMBED_MODEL` / `SKILLMUX_EMBED_MODEL` — overrides `embedding.model`
|
|
470
485
|
- `EMBED_DIMENSION` / `SKILLMUX_EMBED_DIMENSION` — overrides `embedding.dimension`
|
|
471
486
|
- `EMBED_DEVICE` / `EMBED_DTYPE` — overrides local `inference.embedding.device` / `inference.embedding.dtype`
|
|
472
|
-
- `
|
|
473
|
-
- `
|
|
487
|
+
- `RERANK_ENDPOINT` / `SKILLMUX_RERANK_ENDPOINT` — overrides the complete remote `inference.reranker.endpoint`
|
|
488
|
+
- `RERANK_ADAPTER` / `SKILLMUX_RERANK_ADAPTER` — overrides `inference.reranker.adapter`
|
|
489
|
+
- `RERANK_MODEL` / `SKILLMUX_RERANK_MODEL` — overrides `inference.reranker.model`
|
|
474
490
|
- `SKILLMUX_CONFIG` — path to custom `config.toml` (default `~/.config/skillmux/config.toml`)
|
|
475
491
|
- `SKILLMUX_MODELS_DIR` — path to directory storing downloaded local models (default `~/.cache/skillmux/models`, `/models` inside Docker)
|
|
476
492
|
- `PORT` — HTTP listen port (default `3000`, HTTP transport only)
|
|
@@ -482,7 +498,40 @@ All core settings can be overridden via environment variables (handy for Docker)
|
|
|
482
498
|
- `HTTP_RATE_LIMIT_RPM` / `SKILLMUX_HTTP_RATE_LIMIT_RPM` — overrides `server.rate_limit.requests_per_minute`
|
|
483
499
|
- `HTTP_RATE_LIMIT_TRUST_PROXY` / `SKILLMUX_HTTP_RATE_LIMIT_TRUST_PROXY` — overrides `server.rate_limit.trust_proxy` (`"true"` to trust `X-Forwarded-For`)
|
|
484
500
|
|
|
485
|
-
Remote API keys are read from the environment variables named by
|
|
501
|
+
Remote API keys are read independently from the environment variables named by
|
|
502
|
+
`inference.embedding.api_key_env` and `inference.reranker.api_key_env`. Omit
|
|
503
|
+
`api_key_env` for an intentionally unauthenticated endpoint. If it is present,
|
|
504
|
+
the named variable must be non-empty before clients are created and is sent as
|
|
505
|
+
a Bearer token. Secret values never live in the config file or diagnostics.
|
|
506
|
+
|
|
507
|
+
Rerankers use an exact endpoint plus an explicit adapter. `jina-v1` sends
|
|
508
|
+
string documents; `bifrost-v1` sends Bifrost document objects. Skillmux never
|
|
509
|
+
infers an adapter from the URL and never adds or removes path components.
|
|
510
|
+
|
|
511
|
+
Embeddings likewise use an exact endpoint and the OpenAI-compatible
|
|
512
|
+
`{ model, input }` contract. Skillmux never adds `/v1/embeddings` or rewrites
|
|
513
|
+
the configured path or query string. Replace the removed `base_url` setting
|
|
514
|
+
with the complete endpoint, for example `base_url = "http://host"` or
|
|
515
|
+
`base_url = "http://host/v1"` becomes
|
|
516
|
+
`endpoint = "http://host/v1/embeddings"`. The removed `EMBED_BASE_URL`,
|
|
517
|
+
`SKILLMUX_EMBED_BASE_URL`, and `SKILL_ROUTER_EMBED_BASE_URL` variables are
|
|
518
|
+
startup errors with migration guidance.
|
|
519
|
+
|
|
520
|
+
> **Breaking reranker migration:** replace `provider = "infinity"` with
|
|
521
|
+
> `adapter = "jina-v1"`, and replace `base_url` with the complete `endpoint`.
|
|
522
|
+
> The old client appended `/rerank`. For example,
|
|
523
|
+
> `base_url = "http://host/v1"` becomes
|
|
524
|
+
> `endpoint = "http://host/v1/rerank"`; a bare `http://host` becomes
|
|
525
|
+
> `http://host/rerank`. The removed `RERANK_BASE_URL`,
|
|
526
|
+
> `SKILLMUX_RERANK_BASE_URL`, and `SKILL_ROUTER_RERANK_BASE_URL` variables are
|
|
527
|
+
> startup errors with migration guidance.
|
|
528
|
+
|
|
529
|
+
Verified reranker contracts:
|
|
530
|
+
|
|
531
|
+
| Implementation | Tested version | Endpoint | Adapter | Verification |
|
|
532
|
+
| ------------------- | -------------: | -------------------- | ------------ | ----------------------------------------------------------------- |
|
|
533
|
+
| Bifrost | 1.6.6 | `/v1/rerank` | `bifrost-v1` | Live end-to-end request and recorded contract fixture, 2026-07-28 |
|
|
534
|
+
| Jina-style contract | fixture | exact configured URL | `jina-v1` | Automated request/response contract suite |
|
|
486
535
|
|
|
487
536
|
## Benchmarks & Evaluation
|
|
488
537
|
|
|
@@ -506,7 +555,7 @@ Custom policy calibration can also be performed against domain-specific query lo
|
|
|
506
555
|
|
|
507
556
|
<br>
|
|
508
557
|
|
|
509
|
-
The router returns `"outcome": "ambiguous"` when multiple candidate skills meet retrieval confidence thresholds, or when no single candidate dominates by a sufficient score margin. In this state, up to
|
|
558
|
+
The router returns `"outcome": "ambiguous"` when multiple candidate skills meet retrieval confidence thresholds, or when no single candidate dominates by a sufficient score margin. In this state, up to 5 candidate skill summaries (`skill_id`, `title`, `description`) are returned so the calling LLM can choose the exact skill and invoke `fetch_skill`.
|
|
510
559
|
|
|
511
560
|
</details>
|
|
512
561
|
|
|
@@ -7,14 +7,14 @@ timeout_ms = 5000
|
|
|
7
7
|
|
|
8
8
|
[inference.embedding]
|
|
9
9
|
provider = "openai"
|
|
10
|
-
|
|
10
|
+
endpoint = "https://embedding.example.com/v1/embeddings"
|
|
11
11
|
model = "your-embedding-model"
|
|
12
12
|
dimension = 1024
|
|
13
13
|
api_key_env = "EMBEDDING_API_KEY"
|
|
14
14
|
|
|
15
15
|
[inference.reranker]
|
|
16
|
-
|
|
17
|
-
|
|
16
|
+
adapter = "jina-v1"
|
|
17
|
+
endpoint = "https://reranker.example.com/v1/rerank"
|
|
18
18
|
model = "your-reranker-model"
|
|
19
19
|
api_key_env = "RERANKER_API_KEY"
|
|
20
20
|
|
package/docs/configuration.md
CHANGED
|
@@ -51,13 +51,41 @@ Use `skillmux models download` to prefetch models and `skillmux doctor` to verif
|
|
|
51
51
|
|
|
52
52
|
## Remote mode
|
|
53
53
|
|
|
54
|
-
See [`config.remote.example.toml`](../config.remote.example.toml). Embeddings
|
|
54
|
+
See [`config.remote.example.toml`](../config.remote.example.toml). Embeddings
|
|
55
|
+
must implement the OpenAI-compatible `{ model, input }` contract. Configure the
|
|
56
|
+
complete request URL as `inference.embedding.endpoint`; Skillmux does not append
|
|
57
|
+
or rewrite its path or query string. Optional
|
|
58
|
+
reranking uses a versioned wire-protocol adapter and a complete request URL:
|
|
55
59
|
|
|
56
|
-
|
|
60
|
+
```toml
|
|
61
|
+
[inference.reranker]
|
|
62
|
+
adapter = "jina-v1"
|
|
63
|
+
endpoint = "https://reranker.example.com/v1/rerank"
|
|
64
|
+
model = "BAAI/bge-reranker-v2-m3"
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
`jina-v1` sends documents as strings. `bifrost-v1` sends Bifrost document
|
|
68
|
+
objects and requires a Bifrost-style provider-prefixed model name. Skillmux
|
|
69
|
+
does not append `/rerank`, infer the adapter from the URL, or otherwise rewrite
|
|
70
|
+
the endpoint.
|
|
71
|
+
|
|
72
|
+
For embeddings and rerankers independently, omitting `api_key_env` sends no
|
|
73
|
+
`Authorization` header. If `api_key_env` is configured, the named environment
|
|
74
|
+
variable must be non-empty when configuration is loaded; Skillmux sends it as
|
|
75
|
+
a Bearer token. The variable name may appear in diagnostics, but its value
|
|
76
|
+
never does.
|
|
77
|
+
|
|
78
|
+
Remote embedding `dimension` is required. `endpoint`, `api_key_env`, and the
|
|
79
|
+
shared timeout reload live; model, dimension, device, and dtype require a
|
|
80
|
+
restart. Changing only endpoint does not invalidate stored vectors.
|
|
81
|
+
|
|
82
|
+
Reranker adapter and model form the calibration identity. Moving an unchanged
|
|
83
|
+
deployment to another endpoint does not invalidate calibration; changing the
|
|
84
|
+
adapter or model does.
|
|
57
85
|
|
|
58
86
|
## Advanced retrieval
|
|
59
87
|
|
|
60
|
-
Candidate-generation depth
|
|
88
|
+
Candidate-generation depth and agent-context delivery are separate controls:
|
|
61
89
|
|
|
62
90
|
```toml
|
|
63
91
|
[recall]
|
|
@@ -68,7 +96,14 @@ k_vector = 20
|
|
|
68
96
|
candidate_limit = 5
|
|
69
97
|
```
|
|
70
98
|
|
|
71
|
-
|
|
99
|
+
`recall.k_lexical` and `recall.k_vector` control retrieval depth and cost. The
|
|
100
|
+
reranker receives the fused union from both retrieval lanes. Increasing these
|
|
101
|
+
values can improve retrieval recall, but costs more work.
|
|
102
|
+
|
|
103
|
+
`thresholds.candidate_limit` controls agent context: it caps the ambiguous
|
|
104
|
+
candidate list returned to the calling LLM after retrieval, reranking, and
|
|
105
|
+
threshold filtering. It does not change retrieval depth or the matched,
|
|
106
|
+
ambiguous, or no-match classification.
|
|
72
107
|
|
|
73
108
|
Reranker thresholds have no universal default because score distributions are model-specific. When configuring a reranker, provide calibrated `inference.thresholds.match_score`, `inference.thresholds.match_margin`, and `inference.thresholds.candidate_floor`; otherwise configuration is rejected rather than silently applying unsuitable values.
|
|
74
109
|
|
package/docs/schema.json
CHANGED
|
@@ -224,23 +224,31 @@
|
|
|
224
224
|
"type": "object",
|
|
225
225
|
"properties": {
|
|
226
226
|
"provider": { "const": "openai" },
|
|
227
|
-
"
|
|
227
|
+
"endpoint": {
|
|
228
|
+
"type": "string",
|
|
229
|
+
"format": "uri",
|
|
230
|
+
"description": "Complete HTTP(S) OpenAI-compatible embeddings request URL. Skillmux does not modify its path."
|
|
231
|
+
},
|
|
228
232
|
"model": { "type": "string", "minLength": 1 },
|
|
229
233
|
"dimension": { "type": "integer", "minimum": 1 },
|
|
230
234
|
"api_key_env": { "type": "string", "minLength": 1 }
|
|
231
235
|
},
|
|
232
|
-
"required": ["provider", "
|
|
236
|
+
"required": ["provider", "endpoint", "model", "dimension"],
|
|
233
237
|
"additionalProperties": false
|
|
234
238
|
},
|
|
235
239
|
"reranker": {
|
|
236
240
|
"type": "object",
|
|
237
241
|
"properties": {
|
|
238
|
-
"
|
|
239
|
-
"
|
|
242
|
+
"adapter": { "enum": ["jina-v1", "bifrost-v1"] },
|
|
243
|
+
"endpoint": {
|
|
244
|
+
"type": "string",
|
|
245
|
+
"format": "uri",
|
|
246
|
+
"description": "Complete HTTP(S) rerank request URL. Skillmux does not modify its path."
|
|
247
|
+
},
|
|
240
248
|
"model": { "type": "string", "minLength": 1 },
|
|
241
249
|
"api_key_env": { "type": "string", "minLength": 1 }
|
|
242
250
|
},
|
|
243
|
-
"required": ["
|
|
251
|
+
"required": ["adapter", "endpoint", "model"],
|
|
244
252
|
"additionalProperties": false
|
|
245
253
|
},
|
|
246
254
|
"thresholds": {
|
package/package.json
CHANGED
package/src/adapters.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
2
|
import { join } from "node:path";
|
|
3
|
-
import { applyCalibrationRun, getCalibrationRun, listCalibrationRuns, loadDecisionCasesFromFile, openCalibrateDb, runCalibration, type CalibrationResult } from "./calibrate";
|
|
3
|
+
import { applyCalibrationRun, getCalibrationRun, insertCalibrationRun, listCalibrationRuns, loadDecisionCasesFromFile, openCalibrateDb, runCalibration, type CalibrationResult } from "./calibrate";
|
|
4
4
|
import { createClients } from "./clients";
|
|
5
|
-
import { DEFAULT_CONFIG_PATH, expandHome, loadConfig } from "./config";
|
|
5
|
+
import { DEFAULT_CONFIG_PATH, embeddingFingerprint, expandHome, loadConfig, rerankerFingerprint } from "./config";
|
|
6
|
+
import { openIndex } from "./db";
|
|
6
7
|
import { CliError } from "./output";
|
|
7
8
|
import {
|
|
8
9
|
computeHash,
|
|
@@ -18,7 +19,7 @@ import {
|
|
|
18
19
|
type SetConfigResult,
|
|
19
20
|
} from "./config-service";
|
|
20
21
|
import type { ResolvedTarget } from "./context";
|
|
21
|
-
import {
|
|
22
|
+
import { configure, retrieveAndRerank } from "./router-core";
|
|
22
23
|
import type { Config } from "./types";
|
|
23
24
|
|
|
24
25
|
export interface Capabilities {
|
|
@@ -43,7 +44,13 @@ export interface TargetAdapter {
|
|
|
43
44
|
configDiff(): Promise<{ diff: Record<string, { prior: unknown; resulting: unknown }> }>;
|
|
44
45
|
configSet(key: string, rawValStr: string, opts?: { dryRun?: boolean }): Promise<SetConfigResult>;
|
|
45
46
|
configStatus(): Promise<ConfigStatusResponse>;
|
|
46
|
-
calibrateRun(opts?: {
|
|
47
|
+
calibrateRun(opts?: {
|
|
48
|
+
datasetPath?: string;
|
|
49
|
+
minAutoMatchPrecision?: number;
|
|
50
|
+
minRetrievalRecallAtK?: number;
|
|
51
|
+
minDeliveredShortlistRecallAtK?: number;
|
|
52
|
+
minAutoMatchCount?: number;
|
|
53
|
+
}): Promise<{ run_id?: string; result?: CalibrationResult }>;
|
|
47
54
|
calibrateList(): Promise<any[]>;
|
|
48
55
|
calibrateShow(runId: string): Promise<any>;
|
|
49
56
|
calibrateApply(runId: string): Promise<any>;
|
|
@@ -123,26 +130,85 @@ export class LocalAdapter implements TargetAdapter {
|
|
|
123
130
|
return getLocalConfigStatus(this.configPath);
|
|
124
131
|
}
|
|
125
132
|
|
|
126
|
-
async calibrateRun(opts?: {
|
|
133
|
+
async calibrateRun(opts?: {
|
|
134
|
+
datasetPath?: string;
|
|
135
|
+
minAutoMatchPrecision?: number;
|
|
136
|
+
minRetrievalRecallAtK?: number;
|
|
137
|
+
minDeliveredShortlistRecallAtK?: number;
|
|
138
|
+
minAutoMatchCount?: number;
|
|
139
|
+
}): Promise<{ run_id?: string; result?: CalibrationResult }> {
|
|
127
140
|
const config = await loadConfig(this.configPath);
|
|
128
141
|
const datasetFile = opts?.datasetPath ?? join(expandHome(config.state_dir), "queries.json");
|
|
129
142
|
const cases = loadDecisionCasesFromFile(datasetFile);
|
|
130
143
|
const clients = createClients(config);
|
|
144
|
+
configure({ config, clients });
|
|
131
145
|
const result = await runCalibration({
|
|
132
146
|
cases,
|
|
133
|
-
|
|
134
|
-
const
|
|
135
|
-
if (
|
|
136
|
-
|
|
147
|
+
getRankedCandidates: async (query: string) => {
|
|
148
|
+
const result = await retrieveAndRerank({ query, forceLexical: false });
|
|
149
|
+
if (result.retrieval !== "reranked") {
|
|
150
|
+
throw new Error(
|
|
151
|
+
"Calibration requires successful hybrid retrieval and reranking for every query.",
|
|
152
|
+
);
|
|
137
153
|
}
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
154
|
+
return result.candidates.map((candidate) => ({
|
|
155
|
+
skill_id: candidate.skill_id,
|
|
156
|
+
score: candidate.score ?? 0,
|
|
157
|
+
}));
|
|
142
158
|
},
|
|
143
159
|
reranker: clients.rerank,
|
|
160
|
+
candidateLimit: config.thresholds.candidate_limit,
|
|
161
|
+
minAutoMatchPrecision: opts?.minAutoMatchPrecision,
|
|
162
|
+
minRetrievalRecallAtK: opts?.minRetrievalRecallAtK,
|
|
163
|
+
minDeliveredShortlistRecallAtK: opts?.minDeliveredShortlistRecallAtK,
|
|
164
|
+
minAutoMatchCount: opts?.minAutoMatchCount,
|
|
144
165
|
});
|
|
145
|
-
|
|
166
|
+
const fingerprint = rerankerFingerprint(config);
|
|
167
|
+
if (!fingerprint) {
|
|
168
|
+
throw new Error("A configured remote reranker is required to record calibration.");
|
|
169
|
+
}
|
|
170
|
+
const datasetText = await Bun.file(datasetFile).text();
|
|
171
|
+
const indexDb = openIndex(expandHome(config.state_dir));
|
|
172
|
+
let corpusFingerprint: string;
|
|
173
|
+
try {
|
|
174
|
+
const rows = indexDb
|
|
175
|
+
.query("SELECT skill_id, content_sha256 FROM skills ORDER BY skill_id")
|
|
176
|
+
.all();
|
|
177
|
+
corpusFingerprint =
|
|
178
|
+
"vault:" +
|
|
179
|
+
createHash("sha256").update(JSON.stringify(rows)).digest("hex");
|
|
180
|
+
} finally {
|
|
181
|
+
indexDb.close();
|
|
182
|
+
}
|
|
183
|
+
const runId = `run_${crypto.randomUUID()}`;
|
|
184
|
+
const db = openCalibrateDb(expandHome(config.state_dir));
|
|
185
|
+
try {
|
|
186
|
+
insertCalibrationRun(db, {
|
|
187
|
+
run_id: runId,
|
|
188
|
+
created_at: new Date().toISOString(),
|
|
189
|
+
status: result.status,
|
|
190
|
+
reranker_fingerprint: fingerprint,
|
|
191
|
+
embedding_fingerprint: embeddingFingerprint(config),
|
|
192
|
+
corpus_fingerprint: corpusFingerprint,
|
|
193
|
+
dataset_hash: createHash("sha256").update(datasetText).digest("hex"),
|
|
194
|
+
candidate_limit: config.thresholds.candidate_limit,
|
|
195
|
+
min_auto_match_precision: opts?.minAutoMatchPrecision ?? 0.99,
|
|
196
|
+
min_auto_match_count: opts?.minAutoMatchCount ?? 30,
|
|
197
|
+
min_delivered_shortlist_recall_at_k:
|
|
198
|
+
opts?.minDeliveredShortlistRecallAtK ??
|
|
199
|
+
opts?.minRetrievalRecallAtK ??
|
|
200
|
+
0.95,
|
|
201
|
+
min_shortlist_recall_at_5: opts?.minRetrievalRecallAtK ?? 0.95,
|
|
202
|
+
failed_reason: result.failed_reason,
|
|
203
|
+
selected_thresholds: result.selected_thresholds,
|
|
204
|
+
tune_metrics: result.tune_metrics,
|
|
205
|
+
test_metrics: result.test_metrics,
|
|
206
|
+
observations: result.observations,
|
|
207
|
+
});
|
|
208
|
+
} finally {
|
|
209
|
+
db.close();
|
|
210
|
+
}
|
|
211
|
+
return { run_id: runId, result };
|
|
146
212
|
}
|
|
147
213
|
|
|
148
214
|
async calibrateList(): Promise<any[]> {
|
|
@@ -173,7 +239,9 @@ export class LocalAdapter implements TargetAdapter {
|
|
|
173
239
|
try {
|
|
174
240
|
const run = getCalibrationRun(db, runId);
|
|
175
241
|
if (!run) throw new Error(`Calibration run "${runId}" not found`);
|
|
176
|
-
await applyCalibrationRun(db, runId, expandHome(this.configPath), {
|
|
242
|
+
await applyCalibrationRun(db, runId, expandHome(this.configPath), {
|
|
243
|
+
currentRerankerFingerprint: rerankerFingerprint(config),
|
|
244
|
+
});
|
|
177
245
|
return { ok: true, run_id: runId };
|
|
178
246
|
} finally {
|
|
179
247
|
db.close();
|
|
@@ -345,11 +413,23 @@ export class RemoteAdapter implements TargetAdapter {
|
|
|
345
413
|
return data.runtime;
|
|
346
414
|
}
|
|
347
415
|
|
|
348
|
-
async calibrateRun(opts?: {
|
|
416
|
+
async calibrateRun(opts?: {
|
|
417
|
+
datasetPath?: string;
|
|
418
|
+
minAutoMatchPrecision?: number;
|
|
419
|
+
minRetrievalRecallAtK?: number;
|
|
420
|
+
minDeliveredShortlistRecallAtK?: number;
|
|
421
|
+
minAutoMatchCount?: number;
|
|
422
|
+
}): Promise<{ run_id?: string; result?: CalibrationResult }> {
|
|
349
423
|
const { status, data } = await this.fetchJson("/admin/v1/calibrations", {
|
|
350
424
|
method: "POST",
|
|
351
425
|
headers: { "Content-Type": "application/json" },
|
|
352
|
-
body: JSON.stringify({
|
|
426
|
+
body: JSON.stringify({
|
|
427
|
+
dataset_path: opts?.datasetPath,
|
|
428
|
+
min_auto_match_precision: opts?.minAutoMatchPrecision,
|
|
429
|
+
min_retrieval_recall_at_k: opts?.minRetrievalRecallAtK,
|
|
430
|
+
min_delivered_shortlist_recall_at_k: opts?.minDeliveredShortlistRecallAtK,
|
|
431
|
+
min_auto_match_count: opts?.minAutoMatchCount,
|
|
432
|
+
}),
|
|
353
433
|
});
|
|
354
434
|
if (status !== 202) {
|
|
355
435
|
throw new Error(`Remote calibration start failed (${status}): ${data?.message || data}`);
|