@klhapp/skillmux 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/config.example.toml +6 -0
- package/config.remote.example.toml +7 -0
- package/docs/configuration.md +43 -19
- package/docs/releasing.md +10 -3
- package/docs/schema.json +60 -7
- package/package.json +1 -1
- package/src/adapters.ts +9 -2
- package/src/audit.ts +8 -1
- package/src/calibrate.ts +17 -2
- package/src/cli.ts +3 -1
- package/src/clients.ts +19 -0
- package/src/commands/config.ts +13 -0
- package/src/config-service.ts +43 -15
- package/src/config-watcher.ts +3 -0
- package/src/config.ts +183 -54
- package/src/db.ts +15 -3
- package/src/doctor.ts +17 -0
- package/src/eval.ts +69 -12
- package/src/metrics.ts +14 -0
- package/src/router-core.ts +131 -15
- package/src/server.ts +4 -0
- package/src/types.ts +29 -1
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,28 @@ All notable changes to this project are documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [1.5.0](https://github.com/klhq/skillmux/compare/v1.4.1...v1.5.0) (2026-08-17)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
* **config:** add configuration authority, bounded reranking, and degradation resilience ([2626709](https://github.com/klhq/skillmux/commit/262670983b2a7d3674dd02fa5ef90a4a3ebe26c6))
|
|
14
|
+
* **config:** add degradation-aware reranking controls ([1a87ee9](https://github.com/klhq/skillmux/commit/1a87ee90601dc632c127298ebb39be5e8bd12e66))
|
|
15
|
+
* **config:** add output.ambiguous_candidate_limit and evaluation report case details ([7194b28](https://github.com/klhq/skillmux/commit/7194b2874faeebfd89d047063369499ace06785d))
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
### Fixed
|
|
19
|
+
|
|
20
|
+
* **config:** satisfy effective config type checks ([b12693c](https://github.com/klhq/skillmux/commit/b12693ca1ba30d51351d8e5439822ff2f23350df))
|
|
21
|
+
* **routing:** align evaluation with runtime ranking ([ca6e6ca](https://github.com/klhq/skillmux/commit/ca6e6ca847a0604be190e63ac0a177e6e593e817))
|
|
22
|
+
|
|
23
|
+
## [1.4.1](https://github.com/klhq/skillmux/compare/v1.4.0...v1.4.1) (2026-08-04)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
### Fixed
|
|
27
|
+
|
|
28
|
+
* **ci:** publish npm package to GitHub Packages ([#110](https://github.com/klhq/skillmux/issues/110)) ([5de37f2](https://github.com/klhq/skillmux/commit/5de37f22a658187cd270743b24237844513c1e54))
|
|
29
|
+
|
|
8
30
|
## [1.4.0](https://github.com/klhq/skillmux/compare/v1.3.4...v1.4.0) (2026-08-04)
|
|
9
31
|
|
|
10
32
|
|
package/config.example.toml
CHANGED
|
@@ -5,6 +5,12 @@
|
|
|
5
5
|
# which core and project skills appear in those directories.
|
|
6
6
|
vault_path = "~/skills"
|
|
7
7
|
|
|
8
|
+
# Optional: configuration authority policy. Defaults to true.
|
|
9
|
+
# When set to false, behavioral environment variables are ignored and TOML remains
|
|
10
|
+
# authoritative (CLI flags and secret names like api_key_env still resolve).
|
|
11
|
+
# [config]
|
|
12
|
+
# environment_overrides = true
|
|
13
|
+
|
|
8
14
|
# Optional, default []: machine-local override-only vaults, checked before
|
|
9
15
|
# vault_path for a given skill_id. Never holds skillmux.toml or the sync git
|
|
10
16
|
# hook — those always stay in vault_path. See docs/configuration.md#local-vault-overlays.
|
|
@@ -4,6 +4,13 @@
|
|
|
4
4
|
# Keep the canonical vault separate from client skill directories.
|
|
5
5
|
vault_path = "~/skills"
|
|
6
6
|
|
|
7
|
+
# Optional: bounded recall and candidate reranking budgets.
|
|
8
|
+
# k_rerank bounds the maximum number of candidates passed to the reranker adapter.
|
|
9
|
+
# [recall]
|
|
10
|
+
# k_lexical = 20
|
|
11
|
+
# k_vector = 20
|
|
12
|
+
# k_rerank = 10
|
|
13
|
+
|
|
7
14
|
[inference]
|
|
8
15
|
mode = "remote"
|
|
9
16
|
timeout_ms = 5000
|
package/docs/configuration.md
CHANGED
|
@@ -124,35 +124,59 @@ Reranker adapter and model form the calibration identity. Moving an unchanged
|
|
|
124
124
|
deployment to another endpoint does not invalidate calibration; changing the
|
|
125
125
|
adapter or model does.
|
|
126
126
|
|
|
127
|
+
## Configuration authority
|
|
128
|
+
|
|
129
|
+
By default, Skillmux allows namespaced environment variables (`SKILLMUX_*`) to override values from `config.toml`. When deploying in strict environments where the TOML configuration must be authoritative against runtime container environment drift:
|
|
130
|
+
|
|
131
|
+
```toml
|
|
132
|
+
[config]
|
|
133
|
+
environment_overrides = false
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
When `config.environment_overrides = false`:
|
|
137
|
+
- Behavioral environment overrides (e.g. `SKILLMUX_VAULT_PATH`, `SKILLMUX_RECALL_K_RERANK`, `EMBED_MODEL`) are ignored and logged as masked by TOML policy.
|
|
138
|
+
- `SKILLMUX_CONFIG` (config path pointer), CLI flags (`--vault`, `--config`), and named secret variables referenced via `api_key_env` / `token_env` remain fully authoritative.
|
|
139
|
+
- Generic un-namespaced variables (`EMBED_*`, `RERANK_*`, `HTTP_*`, `VAULT_PATH`) are deprecated in 1.x and trigger warnings encouraging the preferred `SKILLMUX_*` namespace.
|
|
140
|
+
|
|
141
|
+
Inspect provenance and active policy with `skillmux config show --sources` or `skillmux doctor`.
|
|
142
|
+
|
|
127
143
|
## Advanced retrieval
|
|
128
144
|
|
|
129
|
-
Candidate-generation depth and agent-context delivery are separate controls:
|
|
145
|
+
Candidate-generation depth, reranking candidate budgets, and agent-context delivery are separate controls:
|
|
130
146
|
|
|
131
147
|
```toml
|
|
132
148
|
[recall]
|
|
133
149
|
k_lexical = 20
|
|
134
150
|
k_vector = 20
|
|
151
|
+
k_rerank = 10
|
|
135
152
|
|
|
136
|
-
[
|
|
137
|
-
|
|
153
|
+
[output]
|
|
154
|
+
ambiguous_candidate_limit = 5
|
|
138
155
|
```
|
|
139
156
|
|
|
140
|
-
`recall.k_lexical` and `recall.k_vector` control
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
157
|
+
- `recall.k_lexical` and `recall.k_vector` control initial candidate generation depth.
|
|
158
|
+
- `recall.k_rerank` bounds the candidate shortlist sent to the reranker adapter (defaults to `10`, cannot exceed `k_lexical + k_vector`).
|
|
159
|
+
- `output.ambiguous_candidate_limit` controls agent context: it caps the ambiguous candidate list returned to the calling LLM after retrieval, reranking, and threshold filtering (`thresholds.candidate_limit` is deprecated in 1.x).
|
|
160
|
+
|
|
161
|
+
### Failure visibility and degraded retrieval
|
|
162
|
+
|
|
163
|
+
When remote embedding or reranking fails or times out, Skillmux gracefully falls back to the strongest surviving retrieval lane (`hybrid` or `lexical`). The response carries structured degradation metadata:
|
|
164
|
+
|
|
165
|
+
```json
|
|
166
|
+
{
|
|
167
|
+
"outcome": "ambiguous",
|
|
168
|
+
"retrieval": "hybrid",
|
|
169
|
+
"degraded_from": "reranked",
|
|
170
|
+
"degradation_reason": "reranker_timeout",
|
|
171
|
+
"candidates": [...]
|
|
172
|
+
}
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Stable degradation reason codes:
|
|
176
|
+
- `embedding_timeout`, `embedding_unavailable`, `embedding_protocol_error`
|
|
177
|
+
- `reranker_timeout`, `reranker_unavailable`, `reranker_protocol_error`
|
|
178
|
+
|
|
179
|
+
These safe reason codes are exposed via MCP and recorded in the audit database without logging credentials or raw upstream response bodies. Degraded retrieval count metrics are tracked under `skill_router_degraded_retrieval_total{stage,reason}`.
|
|
156
180
|
|
|
157
181
|
## HTTP server
|
|
158
182
|
|
package/docs/releasing.md
CHANGED
|
@@ -35,6 +35,8 @@ accidentally republish an existing npm version.
|
|
|
35
35
|
|
|
36
36
|
The release workflow publishes:
|
|
37
37
|
|
|
38
|
+
- `@klhapp/skillmux` to the public npm registry
|
|
39
|
+
- `@klhq/skillmux` to GitHub Packages, linked to this repository
|
|
38
40
|
- `skillmux-linux-amd64`
|
|
39
41
|
- `skillmux-linux-arm64`
|
|
40
42
|
- SHA-256 digests for the Linux binaries in the GitHub Release asset metadata
|
|
@@ -67,11 +69,16 @@ Container images are published to:
|
|
|
67
69
|
- `${DOCKERHUB_USERNAME}/skillmux` on Docker Hub
|
|
68
70
|
|
|
69
71
|
The `production-release` GitHub environment provides the
|
|
70
|
-
`DOCKERHUB_USERNAME` variable and `DOCKERHUB_TOKEN` secret. The
|
|
72
|
+
`DOCKERHUB_USERNAME` variable and `DOCKERHUB_TOKEN` secret. The npmjs job also
|
|
71
73
|
uses this environment as its Trusted Publisher identity; configure npm with
|
|
72
74
|
workflow `release-please.yml` and environment `production-release`. No
|
|
73
|
-
long-lived npm token is required. GitHub Packages uses the workflow's
|
|
74
|
-
`GITHUB_TOKEN
|
|
75
|
+
long-lived npm token is required. The GitHub Packages job uses the workflow's
|
|
76
|
+
scoped `GITHUB_TOKEN` and changes only its runner-local package name to
|
|
77
|
+
`@klhq/skillmux`; the source package remains `@klhapp/skillmux` for npmjs.
|
|
78
|
+
|
|
79
|
+
GitHub initially creates npm packages with private visibility. After the first
|
|
80
|
+
release, open the `@klhq/skillmux` package settings and make it public if it
|
|
81
|
+
should be visible outside the organization.
|
|
75
82
|
Private repositories still publish BuildKit SBOM/provenance with container
|
|
76
83
|
images, but GitHub artifact attestations are skipped because GitHub does not
|
|
77
84
|
support them for user-owned private repositories.
|
package/docs/schema.json
CHANGED
|
@@ -21,6 +21,18 @@
|
|
|
21
21
|
"minLength": 1,
|
|
22
22
|
"maxLength": 512
|
|
23
23
|
},
|
|
24
|
+
"DegradationReason": {
|
|
25
|
+
"type": "string",
|
|
26
|
+
"description": "Safe stable reason code describing why a higher-capability retrieval lane fell back.",
|
|
27
|
+
"enum": [
|
|
28
|
+
"embedding_timeout",
|
|
29
|
+
"embedding_unavailable",
|
|
30
|
+
"embedding_protocol_error",
|
|
31
|
+
"reranker_timeout",
|
|
32
|
+
"reranker_unavailable",
|
|
33
|
+
"reranker_protocol_error"
|
|
34
|
+
]
|
|
35
|
+
},
|
|
24
36
|
"Candidate": {
|
|
25
37
|
"type": "object",
|
|
26
38
|
"description": "One shortlist entry in an ambiguous result.",
|
|
@@ -50,6 +62,8 @@
|
|
|
50
62
|
"properties": {
|
|
51
63
|
"outcome": { "const": "matched" },
|
|
52
64
|
"retrieval": { "enum": ["exact", "reranked"] },
|
|
65
|
+
"degraded_from": { "enum": ["reranked", "hybrid"] },
|
|
66
|
+
"degradation_reason": { "$ref": "#/$defs/DegradationReason" },
|
|
53
67
|
"skill_id": { "$ref": "#/$defs/SkillId" },
|
|
54
68
|
"title": { "type": "string", "minLength": 1, "maxLength": 512 },
|
|
55
69
|
"content_sha256": { "$ref": "#/$defs/Sha256Hex" },
|
|
@@ -83,6 +97,8 @@
|
|
|
83
97
|
"properties": {
|
|
84
98
|
"outcome": { "const": "ambiguous" },
|
|
85
99
|
"retrieval": { "enum": ["exact", "reranked", "hybrid", "lexical"] },
|
|
100
|
+
"degraded_from": { "enum": ["reranked", "hybrid"] },
|
|
101
|
+
"degradation_reason": { "$ref": "#/$defs/DegradationReason" },
|
|
86
102
|
"candidates": {
|
|
87
103
|
"type": "array",
|
|
88
104
|
"items": { "$ref": "#/$defs/Candidate" },
|
|
@@ -99,6 +115,8 @@
|
|
|
99
115
|
"properties": {
|
|
100
116
|
"outcome": { "const": "no_match" },
|
|
101
117
|
"retrieval": { "enum": ["exact", "reranked", "hybrid", "lexical"] },
|
|
118
|
+
"degraded_from": { "enum": ["reranked", "hybrid"] },
|
|
119
|
+
"degradation_reason": { "$ref": "#/$defs/DegradationReason" },
|
|
102
120
|
"message": {
|
|
103
121
|
"type": "string",
|
|
104
122
|
"description": "Fixed guidance string telling the caller to proceed normally.",
|
|
@@ -151,10 +169,26 @@
|
|
|
151
169
|
"type": "object",
|
|
152
170
|
"description": "config.toml, validated at startup. Local ONNX is the default; remote inference is explicit.",
|
|
153
171
|
"properties": {
|
|
172
|
+
"config": {
|
|
173
|
+
"type": "object",
|
|
174
|
+
"properties": {
|
|
175
|
+
"environment_overrides": {
|
|
176
|
+
"type": "boolean",
|
|
177
|
+
"default": true,
|
|
178
|
+
"description": "Whether environment variables override TOML values at runtime."
|
|
179
|
+
}
|
|
180
|
+
},
|
|
181
|
+
"additionalProperties": false
|
|
182
|
+
},
|
|
154
183
|
"vault_path": {
|
|
155
184
|
"type": "string",
|
|
156
|
-
"description": "Skill vault root. Default:
|
|
157
|
-
"default": "
|
|
185
|
+
"description": "Skill vault root. Default: ~/skills",
|
|
186
|
+
"default": "~/skills"
|
|
187
|
+
},
|
|
188
|
+
"local_vault_paths": {
|
|
189
|
+
"type": "array",
|
|
190
|
+
"items": { "type": "string" },
|
|
191
|
+
"default": []
|
|
158
192
|
},
|
|
159
193
|
"state_dir": {
|
|
160
194
|
"type": "string",
|
|
@@ -165,30 +199,43 @@
|
|
|
165
199
|
"type": "object",
|
|
166
200
|
"properties": {
|
|
167
201
|
"k_lexical": { "type": "integer", "minimum": 1, "maximum": 100, "default": 20 },
|
|
168
|
-
"k_vector": { "type": "integer", "minimum": 1, "maximum": 100, "default": 20 }
|
|
202
|
+
"k_vector": { "type": "integer", "minimum": 1, "maximum": 100, "default": 20 },
|
|
203
|
+
"k_rerank": { "type": "integer", "minimum": 1, "maximum": 200, "default": 10 }
|
|
169
204
|
},
|
|
170
205
|
"required": ["k_lexical", "k_vector"],
|
|
171
206
|
"additionalProperties": false
|
|
172
207
|
},
|
|
208
|
+
"output": {
|
|
209
|
+
"type": "object",
|
|
210
|
+
"properties": {
|
|
211
|
+
"ambiguous_candidate_limit": {
|
|
212
|
+
"type": "integer",
|
|
213
|
+
"minimum": 1,
|
|
214
|
+
"maximum": 50,
|
|
215
|
+
"default": 5,
|
|
216
|
+
"description": "Maximum number of candidates returned on ambiguous outcomes."
|
|
217
|
+
}
|
|
218
|
+
},
|
|
219
|
+
"additionalProperties": false
|
|
220
|
+
},
|
|
173
221
|
"thresholds": {
|
|
174
222
|
"type": "object",
|
|
175
|
-
"description": "Caller-facing shortlist policy. Reranker thresholds live under inference.thresholds.",
|
|
223
|
+
"description": "Caller-facing shortlist policy (deprecated in favor of output.ambiguous_candidate_limit). Reranker thresholds live under inference.thresholds.",
|
|
176
224
|
"properties": {
|
|
177
225
|
"candidate_limit": {
|
|
178
226
|
"type": "integer",
|
|
179
227
|
"minimum": 1,
|
|
180
228
|
"maximum": 50,
|
|
181
229
|
"default": 5,
|
|
182
|
-
"description": "
|
|
230
|
+
"description": "Deprecated: use output.ambiguous_candidate_limit instead."
|
|
183
231
|
}
|
|
184
232
|
},
|
|
185
|
-
"required": ["candidate_limit"],
|
|
186
233
|
"additionalProperties": false
|
|
187
234
|
},
|
|
188
235
|
"inference": { "$ref": "#/$defs/InferenceConfig" },
|
|
189
236
|
"server": { "$ref": "#/$defs/ServerConfig" }
|
|
190
237
|
},
|
|
191
|
-
"required": ["vault_path", "state_dir", "recall", "
|
|
238
|
+
"required": ["vault_path", "state_dir", "recall", "inference"],
|
|
192
239
|
"additionalProperties": false
|
|
193
240
|
},
|
|
194
241
|
"InferenceConfig": {
|
|
@@ -295,6 +342,12 @@
|
|
|
295
342
|
"query": { "type": "string" },
|
|
296
343
|
"outcome": { "type": "string", "enum": ["matched", "ambiguous", "no_match"] },
|
|
297
344
|
"retrieval": { "enum": ["exact", "reranked", "hybrid", "lexical"] },
|
|
345
|
+
"degraded_from": {
|
|
346
|
+
"oneOf": [{ "enum": ["reranked", "hybrid"] }, { "type": "null" }]
|
|
347
|
+
},
|
|
348
|
+
"degradation_reason": {
|
|
349
|
+
"oneOf": [{ "$ref": "#/$defs/DegradationReason" }, { "type": "null" }]
|
|
350
|
+
},
|
|
298
351
|
"candidates": {
|
|
299
352
|
"type": "array",
|
|
300
353
|
"description": "skill_id + score of every reranked (or FTS5-ranked) candidate considered, best first.",
|
package/package.json
CHANGED
package/src/adapters.ts
CHANGED
|
@@ -138,6 +138,8 @@ export class LocalAdapter implements TargetAdapter {
|
|
|
138
138
|
minAutoMatchCount?: number;
|
|
139
139
|
}): Promise<{ run_id?: string; result?: CalibrationResult }> {
|
|
140
140
|
const config = await loadConfig(this.configPath);
|
|
141
|
+
const candidateLimit =
|
|
142
|
+
config.output?.ambiguous_candidate_limit ?? config.thresholds.candidate_limit ?? 5;
|
|
141
143
|
const datasetFile = opts?.datasetPath ?? join(expandHome(config.state_dir), "queries.json");
|
|
142
144
|
const indexDb = openIndex(expandHome(config.state_dir));
|
|
143
145
|
let indexedSkills: Array<{ skill_id: string; content_sha256: string }>;
|
|
@@ -171,7 +173,7 @@ export class LocalAdapter implements TargetAdapter {
|
|
|
171
173
|
}));
|
|
172
174
|
},
|
|
173
175
|
reranker: clients.rerank,
|
|
174
|
-
candidateLimit
|
|
176
|
+
candidateLimit,
|
|
175
177
|
minAutoMatchPrecision: opts?.minAutoMatchPrecision,
|
|
176
178
|
minRetrievalRecallAtK: opts?.minRetrievalRecallAtK,
|
|
177
179
|
minDeliveredShortlistRecallAtK: opts?.minDeliveredShortlistRecallAtK,
|
|
@@ -194,7 +196,12 @@ export class LocalAdapter implements TargetAdapter {
|
|
|
194
196
|
corpus_fingerprint: corpusFingerprint,
|
|
195
197
|
dataset_hash: createHash("sha256").update(datasetText).digest("hex"),
|
|
196
198
|
dataset_provenance: summarizeDatasetProvenance(cases),
|
|
197
|
-
|
|
199
|
+
recall_settings: {
|
|
200
|
+
k_lexical: config.recall.k_lexical,
|
|
201
|
+
k_vector: config.recall.k_vector,
|
|
202
|
+
k_rerank: config.recall.k_rerank ?? Math.min(10, config.recall.k_lexical + config.recall.k_vector),
|
|
203
|
+
},
|
|
204
|
+
candidate_limit: candidateLimit,
|
|
198
205
|
min_auto_match_precision: opts?.minAutoMatchPrecision ?? 0.99,
|
|
199
206
|
min_auto_match_count: opts?.minAutoMatchCount ?? 30,
|
|
200
207
|
min_delivered_shortlist_recall_at_k:
|
package/src/audit.ts
CHANGED
|
@@ -2,7 +2,7 @@ import type { AuditRow } from "./types";
|
|
|
2
2
|
|
|
3
3
|
/** Shape an audit row to exactly the schema's AuditRow fields — nothing extra survives. */
|
|
4
4
|
export function buildAuditRow(row: AuditRow): AuditRow {
|
|
5
|
-
|
|
5
|
+
const built: AuditRow = {
|
|
6
6
|
id: row.id,
|
|
7
7
|
ts: row.ts,
|
|
8
8
|
query: row.query,
|
|
@@ -12,4 +12,11 @@ export function buildAuditRow(row: AuditRow): AuditRow {
|
|
|
12
12
|
selected_skill_id: row.selected_skill_id,
|
|
13
13
|
latency_ms: row.latency_ms,
|
|
14
14
|
};
|
|
15
|
+
if (row.degraded_from !== undefined && row.degraded_from !== null) {
|
|
16
|
+
built.degraded_from = row.degraded_from;
|
|
17
|
+
}
|
|
18
|
+
if (row.degradation_reason !== undefined && row.degradation_reason !== null) {
|
|
19
|
+
built.degradation_reason = row.degradation_reason;
|
|
20
|
+
}
|
|
21
|
+
return built;
|
|
15
22
|
}
|
package/src/calibrate.ts
CHANGED
|
@@ -867,6 +867,11 @@ export interface CalibrationRunRecord {
|
|
|
867
867
|
embedding_fingerprint: string;
|
|
868
868
|
corpus_fingerprint: string;
|
|
869
869
|
dataset_hash: string;
|
|
870
|
+
recall_settings?: {
|
|
871
|
+
k_lexical: number;
|
|
872
|
+
k_vector: number;
|
|
873
|
+
k_rerank: number;
|
|
874
|
+
};
|
|
870
875
|
dataset_provenance?: DatasetProvenanceSummary;
|
|
871
876
|
candidate_limit: number;
|
|
872
877
|
attempt_count?: number;
|
|
@@ -963,6 +968,9 @@ export function openCalibrateDb(stateDir: string): Database {
|
|
|
963
968
|
if (!columns.some((column) => column.name === "imported_labelled_case_count")) {
|
|
964
969
|
db.run("ALTER TABLE calibration_runs ADD COLUMN imported_labelled_case_count INTEGER NOT NULL DEFAULT 0");
|
|
965
970
|
}
|
|
971
|
+
if (!columns.some((column) => column.name === "recall_settings")) {
|
|
972
|
+
db.run("ALTER TABLE calibration_runs ADD COLUMN recall_settings TEXT NOT NULL DEFAULT '{}'");
|
|
973
|
+
}
|
|
966
974
|
return db;
|
|
967
975
|
}
|
|
968
976
|
|
|
@@ -980,8 +988,9 @@ export function insertCalibrationRun(db: Database, run: CalibrationRunRecord): v
|
|
|
980
988
|
attempt_count, min_auto_match_precision, min_auto_match_count,
|
|
981
989
|
min_delivered_shortlist_recall_at_k, min_shortlist_recall_at_5, failed_reason,
|
|
982
990
|
selected_thresholds, tune_metrics, test_metrics, observations,
|
|
983
|
-
dataset_provenance, human_labelled_case_count, imported_labelled_case_count
|
|
984
|
-
|
|
991
|
+
dataset_provenance, human_labelled_case_count, imported_labelled_case_count,
|
|
992
|
+
recall_settings
|
|
993
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
985
994
|
[
|
|
986
995
|
run.run_id,
|
|
987
996
|
run.created_at,
|
|
@@ -1004,6 +1013,7 @@ export function insertCalibrationRun(db: Database, run: CalibrationRunRecord): v
|
|
|
1004
1013
|
JSON.stringify(run.dataset_provenance ?? {}),
|
|
1005
1014
|
run.dataset_provenance?.human_labelled_case_count ?? 0,
|
|
1006
1015
|
run.dataset_provenance?.imported_labelled_case_count ?? 0,
|
|
1016
|
+
JSON.stringify(run.recall_settings ?? {}),
|
|
1007
1017
|
],
|
|
1008
1018
|
);
|
|
1009
1019
|
}
|
|
@@ -1030,6 +1040,7 @@ interface RawCalibrationRow {
|
|
|
1030
1040
|
dataset_provenance: string;
|
|
1031
1041
|
human_labelled_case_count: number;
|
|
1032
1042
|
imported_labelled_case_count: number;
|
|
1043
|
+
recall_settings: string;
|
|
1033
1044
|
}
|
|
1034
1045
|
|
|
1035
1046
|
function parseMetrics(json: string): CalibrationMetrics {
|
|
@@ -1064,6 +1075,10 @@ function rowToRecord(row: RawCalibrationRow): CalibrationRunRecord {
|
|
|
1064
1075
|
embedding_fingerprint: row.embedding_fingerprint,
|
|
1065
1076
|
corpus_fingerprint: row.corpus_fingerprint,
|
|
1066
1077
|
dataset_hash: row.dataset_hash,
|
|
1078
|
+
recall_settings:
|
|
1079
|
+
Object.keys(JSON.parse(row.recall_settings) as object).length > 0
|
|
1080
|
+
? JSON.parse(row.recall_settings) as CalibrationRunRecord["recall_settings"]
|
|
1081
|
+
: undefined,
|
|
1067
1082
|
dataset_provenance:
|
|
1068
1083
|
Object.keys(JSON.parse(row.dataset_provenance) as object).length > 0
|
|
1069
1084
|
? JSON.parse(row.dataset_provenance) as DatasetProvenanceSummary
|
package/src/cli.ts
CHANGED
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
} from "./config";
|
|
16
16
|
import { openIndex } from "./db";
|
|
17
17
|
import { diagnose } from "./doctor";
|
|
18
|
+
import { getEffectiveConfig } from "./config-service";
|
|
18
19
|
import { evalVault } from "./eval";
|
|
19
20
|
import {
|
|
20
21
|
assessClientReadiness,
|
|
@@ -745,7 +746,8 @@ async function runEval(options: { isJson: boolean }): Promise<void> {
|
|
|
745
746
|
}
|
|
746
747
|
|
|
747
748
|
async function runDoctor(options: { isJson: boolean }): Promise<void> {
|
|
748
|
-
const
|
|
749
|
+
const effective = await getEffectiveConfig(resolveConfigPath());
|
|
750
|
+
const report = await diagnose(effective.effective, process.env, effective.sources);
|
|
749
751
|
emitSuccess({ isJson: options.isJson }, report, () => {
|
|
750
752
|
console.log(`version: ${report.version}`);
|
|
751
753
|
console.log(`runtime: ${report.runtime}`);
|
package/src/clients.ts
CHANGED
|
@@ -212,6 +212,17 @@ async function fetchRerankerScores(
|
|
|
212
212
|
});
|
|
213
213
|
} catch (error) {
|
|
214
214
|
if (error instanceof RemoteInferenceError) throw error;
|
|
215
|
+
const isTimeout =
|
|
216
|
+
(error as { name?: string })?.name === "TimeoutError" ||
|
|
217
|
+
(error as { name?: string })?.name === "AbortError" ||
|
|
218
|
+
String(error).toLowerCase().includes("timeout") ||
|
|
219
|
+
String(error).toLowerCase().includes("aborted");
|
|
220
|
+
if (isTimeout) {
|
|
221
|
+
throw new RemoteInferenceError(
|
|
222
|
+
"availability",
|
|
223
|
+
`reranker adapter "${reranker.adapter}" request timed out`,
|
|
224
|
+
);
|
|
225
|
+
}
|
|
215
226
|
throw new RemoteInferenceError(
|
|
216
227
|
"availability",
|
|
217
228
|
`reranker adapter "${reranker.adapter}" request failed`,
|
|
@@ -302,6 +313,14 @@ export function createClients(config: Config): Clients {
|
|
|
302
313
|
});
|
|
303
314
|
} catch (error) {
|
|
304
315
|
if (error instanceof RemoteInferenceError) throw error;
|
|
316
|
+
const isTimeout =
|
|
317
|
+
(error as { name?: string })?.name === "TimeoutError" ||
|
|
318
|
+
(error as { name?: string })?.name === "AbortError" ||
|
|
319
|
+
String(error).toLowerCase().includes("timeout") ||
|
|
320
|
+
String(error).toLowerCase().includes("aborted");
|
|
321
|
+
if (isTimeout) {
|
|
322
|
+
throw new RemoteInferenceError("availability", "embedding endpoint request timed out");
|
|
323
|
+
}
|
|
305
324
|
throw new RemoteInferenceError("availability", "embedding endpoint request failed");
|
|
306
325
|
}
|
|
307
326
|
if (!response.ok) throw httpFailure("embedding endpoint", response.status);
|
package/src/commands/config.ts
CHANGED
|
@@ -124,9 +124,22 @@ export async function handleConfigCommand(
|
|
|
124
124
|
}
|
|
125
125
|
|
|
126
126
|
if (sub === "show") {
|
|
127
|
+
const withSources = args.includes("--sources");
|
|
127
128
|
const data = await adapter.getConfigShow();
|
|
128
129
|
emitSuccess({ isJson: ctx.isJson, target: ctx.target }, data, () => {
|
|
129
130
|
renderTargetBanner(ctx.target);
|
|
131
|
+
if (withSources) {
|
|
132
|
+
const policy =
|
|
133
|
+
data.effective.config?.environment_overrides === false
|
|
134
|
+
? "strict (TOML authoritative)"
|
|
135
|
+
: "permissive (environment overrides enabled)";
|
|
136
|
+
console.log(`Policy: ${policy}`);
|
|
137
|
+
console.log("\nSources:");
|
|
138
|
+
for (const [k, src] of Object.entries(data.sources)) {
|
|
139
|
+
console.log(` ${k}: ${src}`);
|
|
140
|
+
}
|
|
141
|
+
console.log("\nEffective Configuration:");
|
|
142
|
+
}
|
|
130
143
|
console.log(JSON.stringify(data.effective, null, 2));
|
|
131
144
|
});
|
|
132
145
|
return;
|
package/src/config-service.ts
CHANGED
|
@@ -55,9 +55,12 @@ export const RESTART_REQUIRED_KEYS = [
|
|
|
55
55
|
];
|
|
56
56
|
|
|
57
57
|
export const RELOADABLE_KEYS = [
|
|
58
|
+
"config.environment_overrides",
|
|
58
59
|
"vault_path",
|
|
59
60
|
"recall.k_lexical",
|
|
60
61
|
"recall.k_vector",
|
|
62
|
+
"recall.k_rerank",
|
|
63
|
+
"output.ambiguous_candidate_limit",
|
|
61
64
|
"thresholds.candidate_limit",
|
|
62
65
|
"thresholds.match_score",
|
|
63
66
|
"thresholds.match_margin",
|
|
@@ -138,12 +141,16 @@ export async function getEffectiveConfig(configPath?: string): Promise<{
|
|
|
138
141
|
}
|
|
139
142
|
|
|
140
143
|
const sources: ConfigSourceMap = {};
|
|
144
|
+
const allowEnvOverrides = effective.config?.environment_overrides !== false;
|
|
141
145
|
|
|
142
146
|
const allKeys = [
|
|
147
|
+
"config.environment_overrides",
|
|
143
148
|
"vault_path",
|
|
144
149
|
"state_dir",
|
|
145
150
|
"recall.k_lexical",
|
|
146
151
|
"recall.k_vector",
|
|
152
|
+
"recall.k_rerank",
|
|
153
|
+
"output.ambiguous_candidate_limit",
|
|
147
154
|
"thresholds.candidate_limit",
|
|
148
155
|
"thresholds.match_score",
|
|
149
156
|
"thresholds.match_margin",
|
|
@@ -164,6 +171,7 @@ export async function getEffectiveConfig(configPath?: string): Promise<{
|
|
|
164
171
|
"inference.timeout_ms",
|
|
165
172
|
"server.auth_enabled",
|
|
166
173
|
"server.auth_token_env",
|
|
174
|
+
"server.allowed_origins",
|
|
167
175
|
"server.admin.enabled",
|
|
168
176
|
"server.admin.token_env",
|
|
169
177
|
"server.hostname",
|
|
@@ -173,7 +181,7 @@ export async function getEffectiveConfig(configPath?: string): Promise<{
|
|
|
173
181
|
];
|
|
174
182
|
|
|
175
183
|
for (const key of allKeys) {
|
|
176
|
-
if (isEnvMasked(key)) {
|
|
184
|
+
if (isEnvMasked(key, allowEnvOverrides)) {
|
|
177
185
|
sources[key] = "environment";
|
|
178
186
|
} else if (getNestedValue(rawToml, key) !== undefined) {
|
|
179
187
|
sources[key] = "toml";
|
|
@@ -185,21 +193,35 @@ export async function getEffectiveConfig(configPath?: string): Promise<{
|
|
|
185
193
|
return { effective, sources, rawToml };
|
|
186
194
|
}
|
|
187
195
|
|
|
188
|
-
export function isEnvMasked(key: string): boolean {
|
|
189
|
-
if (
|
|
190
|
-
if (key === "
|
|
196
|
+
export function isEnvMasked(key: string, allowEnvOverrides: boolean = true): boolean {
|
|
197
|
+
if (!allowEnvOverrides) return false;
|
|
198
|
+
if (key === "vault_path" && (process.env.SKILLMUX_VAULT_PATH || process.env.SKILL_ROUTER_VAULT_PATH || process.env.VAULT_PATH)) return true;
|
|
199
|
+
if (key === "state_dir" && (process.env.SKILLMUX_STATE_DIR || process.env.SKILL_ROUTER_STATE_DIR || process.env.STATE_DIR)) return true;
|
|
200
|
+
if (key === "recall.k_lexical" && (process.env.SKILLMUX_RECALL_K_LEXICAL || process.env.RECALL_K_LEXICAL)) return true;
|
|
201
|
+
if (key === "recall.k_vector" && (process.env.SKILLMUX_RECALL_K_VECTOR || process.env.RECALL_K_VECTOR)) return true;
|
|
202
|
+
if (key === "recall.k_rerank" && (process.env.SKILLMUX_RECALL_K_RERANK || process.env.RECALL_K_RERANK)) return true;
|
|
203
|
+
if (
|
|
204
|
+
(key === "output.ambiguous_candidate_limit" || key === "thresholds.candidate_limit") &&
|
|
205
|
+
(process.env.SKILLMUX_OUTPUT_AMBIGUOUS_CANDIDATE_LIMIT ||
|
|
206
|
+
process.env.AMBIGUOUS_CANDIDATE_LIMIT ||
|
|
207
|
+
process.env.SKILLMUX_CANDIDATE_LIMIT ||
|
|
208
|
+
process.env.CANDIDATE_LIMIT)
|
|
209
|
+
) {
|
|
210
|
+
return true;
|
|
211
|
+
}
|
|
191
212
|
if (key === "inference.models_dir" && (process.env.SKILLMUX_MODELS_DIR || process.env.SKILL_ROUTER_MODELS_DIR)) return true;
|
|
192
|
-
if (key === "inference.embedding.device" && process.env.EMBED_DEVICE) return true;
|
|
193
|
-
if (key === "inference.embedding.dtype" && process.env.EMBED_DTYPE) return true;
|
|
194
|
-
if (key === "inference.embedding.endpoint" && (process.env.SKILLMUX_EMBED_ENDPOINT || process.env.EMBED_ENDPOINT)) return true;
|
|
195
|
-
if (key === "inference.embedding.model" && (process.env.SKILLMUX_EMBED_MODEL || process.env.EMBED_MODEL)) return true;
|
|
196
|
-
if (key === "inference.embedding.dimension" && (process.env.SKILLMUX_EMBED_DIMENSION || process.env.EMBED_DIMENSION)) return true;
|
|
197
|
-
if (key === "inference.reranker.adapter" && (process.env.SKILLMUX_RERANK_ADAPTER || process.env.RERANK_ADAPTER)) return true;
|
|
198
|
-
if (key === "inference.reranker.endpoint" && (process.env.SKILLMUX_RERANK_ENDPOINT || process.env.RERANK_ENDPOINT)) return true;
|
|
199
|
-
if (key === "inference.reranker.model" && (process.env.SKILLMUX_RERANK_MODEL || process.env.RERANK_MODEL)) return true;
|
|
200
|
-
if (key === "server.auth_enabled" && process.env.HTTP_AUTH_ENABLED) return true;
|
|
201
|
-
if (key === "server.auth_token_env" && process.env.HTTP_AUTH_TOKEN_ENV) return true;
|
|
202
|
-
if (key === "server.
|
|
213
|
+
if (key === "inference.embedding.device" && (process.env.SKILLMUX_EMBED_DEVICE || process.env.EMBED_DEVICE)) return true;
|
|
214
|
+
if (key === "inference.embedding.dtype" && (process.env.SKILLMUX_EMBED_DTYPE || process.env.EMBED_DTYPE)) return true;
|
|
215
|
+
if (key === "inference.embedding.endpoint" && (process.env.SKILLMUX_EMBED_ENDPOINT || process.env.SKILL_ROUTER_EMBED_ENDPOINT || process.env.EMBED_ENDPOINT)) return true;
|
|
216
|
+
if (key === "inference.embedding.model" && (process.env.SKILLMUX_EMBED_MODEL || process.env.SKILL_ROUTER_EMBED_MODEL || process.env.EMBED_MODEL)) return true;
|
|
217
|
+
if (key === "inference.embedding.dimension" && (process.env.SKILLMUX_EMBED_DIMENSION || process.env.SKILL_ROUTER_EMBED_DIMENSION || process.env.EMBED_DIMENSION)) return true;
|
|
218
|
+
if (key === "inference.reranker.adapter" && (process.env.SKILLMUX_RERANK_ADAPTER || process.env.SKILL_ROUTER_RERANK_ADAPTER || process.env.RERANK_ADAPTER)) return true;
|
|
219
|
+
if (key === "inference.reranker.endpoint" && (process.env.SKILLMUX_RERANK_ENDPOINT || process.env.SKILL_ROUTER_RERANK_ENDPOINT || process.env.RERANK_ENDPOINT)) return true;
|
|
220
|
+
if (key === "inference.reranker.model" && (process.env.SKILLMUX_RERANK_MODEL || process.env.SKILL_ROUTER_RERANK_MODEL || process.env.RERANK_MODEL)) return true;
|
|
221
|
+
if (key === "server.auth_enabled" && (process.env.SKILLMUX_HTTP_AUTH_ENABLED || process.env.HTTP_AUTH_ENABLED)) return true;
|
|
222
|
+
if (key === "server.auth_token_env" && (process.env.SKILLMUX_HTTP_AUTH_TOKEN_ENV || process.env.HTTP_AUTH_TOKEN_ENV)) return true;
|
|
223
|
+
if (key === "server.allowed_origins" && (process.env.SKILLMUX_HTTP_ALLOWED_ORIGINS || process.env.HTTP_ALLOWED_ORIGINS)) return true;
|
|
224
|
+
if (key === "server.hostname" && (process.env.SKILLMUX_HTTP_HOSTNAME || process.env.HTTP_HOSTNAME)) return true;
|
|
203
225
|
if (key === "server.rate_limit.enabled" && (process.env.SKILLMUX_HTTP_RATE_LIMIT_ENABLED || process.env.HTTP_RATE_LIMIT_ENABLED)) return true;
|
|
204
226
|
if (key === "server.rate_limit.requests_per_minute" && (process.env.SKILLMUX_HTTP_RATE_LIMIT_RPM || process.env.HTTP_RATE_LIMIT_RPM)) return true;
|
|
205
227
|
if (key === "server.rate_limit.trust_proxy" && (process.env.SKILLMUX_HTTP_RATE_LIMIT_TRUST_PROXY || process.env.HTTP_RATE_LIMIT_TRUST_PROXY)) return true;
|
|
@@ -208,10 +230,13 @@ export function isEnvMasked(key: string): boolean {
|
|
|
208
230
|
|
|
209
231
|
export function validateDottedKey(key: string): void {
|
|
210
232
|
const allowed = new Set([
|
|
233
|
+
"config.environment_overrides",
|
|
211
234
|
"vault_path",
|
|
212
235
|
"state_dir",
|
|
213
236
|
"recall.k_lexical",
|
|
214
237
|
"recall.k_vector",
|
|
238
|
+
"recall.k_rerank",
|
|
239
|
+
"output.ambiguous_candidate_limit",
|
|
215
240
|
"thresholds.candidate_limit",
|
|
216
241
|
"thresholds.match_score",
|
|
217
242
|
"thresholds.match_margin",
|
|
@@ -253,6 +278,8 @@ export function parseDottedValue(key: string, valueStr: string): unknown {
|
|
|
253
278
|
const numberKeys = new Set([
|
|
254
279
|
"recall.k_lexical",
|
|
255
280
|
"recall.k_vector",
|
|
281
|
+
"recall.k_rerank",
|
|
282
|
+
"output.ambiguous_candidate_limit",
|
|
256
283
|
"thresholds.candidate_limit",
|
|
257
284
|
"thresholds.match_score",
|
|
258
285
|
"thresholds.match_margin",
|
|
@@ -271,6 +298,7 @@ export function parseDottedValue(key: string, valueStr: string): unknown {
|
|
|
271
298
|
}
|
|
272
299
|
|
|
273
300
|
const booleanKeys = new Set([
|
|
301
|
+
"config.environment_overrides",
|
|
274
302
|
"server.auth_enabled",
|
|
275
303
|
"server.admin.enabled",
|
|
276
304
|
"server.rate_limit.enabled",
|
package/src/config-watcher.ts
CHANGED
|
@@ -11,12 +11,15 @@ import type { Config } from "./types";
|
|
|
11
11
|
// ---------------------------------------------------------------------------
|
|
12
12
|
|
|
13
13
|
export const LIVE_RELOAD_KEYS = new Set([
|
|
14
|
+
"config.environment_overrides",
|
|
14
15
|
"inference.thresholds.match_score",
|
|
15
16
|
"inference.thresholds.match_margin",
|
|
16
17
|
"inference.thresholds.candidate_floor",
|
|
17
18
|
"inference.calibration.run_id",
|
|
18
19
|
"recall.k_lexical",
|
|
19
20
|
"recall.k_vector",
|
|
21
|
+
"recall.k_rerank",
|
|
22
|
+
"output.ambiguous_candidate_limit",
|
|
20
23
|
"thresholds.candidate_limit",
|
|
21
24
|
"inference.embedding.endpoint",
|
|
22
25
|
"inference.embedding.api_key_env",
|