@klhapp/skillmux 1.0.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/README.md +83 -29
- package/config.remote.example.toml +6 -4
- package/docs/calibration.md +106 -0
- package/docs/configuration.md +51 -7
- package/docs/schema.json +13 -5
- package/package.json +2 -1
- package/src/adapters.ts +111 -38
- package/src/calibrate.ts +623 -125
- package/src/cli.ts +56 -6
- package/src/clients.ts +264 -48
- package/src/config-service.ts +25 -8
- package/src/config-watcher.ts +7 -0
- package/src/config.ts +106 -25
- package/src/dataset-generator.ts +75 -96
- package/src/decision.ts +4 -1
- package/src/doctor.ts +16 -5
- package/src/eval.ts +2 -1
- package/src/router-core.ts +73 -42
- package/src/server.ts +9 -66
- package/src/types.ts +3 -3
package/src/config.ts
CHANGED
|
@@ -48,14 +48,14 @@ const configSchema = z.object({
|
|
|
48
48
|
timeout_ms: z.number().int().min(100),
|
|
49
49
|
embedding: z.object({
|
|
50
50
|
provider: z.literal("openai"),
|
|
51
|
-
|
|
51
|
+
endpoint: z.url(),
|
|
52
52
|
model: z.string().min(1),
|
|
53
53
|
dimension: z.number().int().positive(),
|
|
54
54
|
api_key_env: z.string().min(1).optional(),
|
|
55
55
|
}).strict(),
|
|
56
56
|
reranker: z.object({
|
|
57
|
-
|
|
58
|
-
|
|
57
|
+
adapter: z.enum(["jina-v1", "bifrost-v1"]),
|
|
58
|
+
endpoint: z.url(),
|
|
59
59
|
model: z.string().min(1),
|
|
60
60
|
api_key_env: z.string().min(1).optional(),
|
|
61
61
|
}).strict().optional(),
|
|
@@ -133,6 +133,12 @@ export function embeddingFingerprint(config: Config): string {
|
|
|
133
133
|
return `${implementation}:${inference.embedding.model}:${inference.embedding.dimension}`;
|
|
134
134
|
}
|
|
135
135
|
|
|
136
|
+
export function rerankerFingerprint(config: Config): string | undefined {
|
|
137
|
+
const inference = config.inference;
|
|
138
|
+
if (inference.mode !== "remote" || !inference.reranker) return undefined;
|
|
139
|
+
return `remote:${inference.reranker.adapter}:${inference.reranker.model}`;
|
|
140
|
+
}
|
|
141
|
+
|
|
136
142
|
export function expandHome(path: string): string {
|
|
137
143
|
return path.startsWith("~") ? join(homedir(), path.slice(1)) : path;
|
|
138
144
|
}
|
|
@@ -183,6 +189,30 @@ export function resolveConfigPath(path?: string): string {
|
|
|
183
189
|
|
|
184
190
|
export async function loadConfig(path?: string): Promise<Config> {
|
|
185
191
|
migrateLegacyPaths();
|
|
192
|
+
const removedRerankerEnv = [
|
|
193
|
+
"SKILLMUX_RERANK_BASE_URL",
|
|
194
|
+
"SKILL_ROUTER_RERANK_BASE_URL",
|
|
195
|
+
"RERANK_BASE_URL",
|
|
196
|
+
].find((name) => process.env[name] !== undefined);
|
|
197
|
+
if (removedRerankerEnv) {
|
|
198
|
+
throw new Error(
|
|
199
|
+
`${removedRerankerEnv} is no longer supported. Configure ` +
|
|
200
|
+
"inference.reranker.endpoint with the complete request URL and " +
|
|
201
|
+
'inference.reranker.adapter (for example, "jina-v1"). The old client appended /rerank.',
|
|
202
|
+
);
|
|
203
|
+
}
|
|
204
|
+
const removedEmbeddingEnv = [
|
|
205
|
+
"SKILLMUX_EMBED_BASE_URL",
|
|
206
|
+
"SKILL_ROUTER_EMBED_BASE_URL",
|
|
207
|
+
"EMBED_BASE_URL",
|
|
208
|
+
].find((name) => process.env[name] !== undefined);
|
|
209
|
+
if (removedEmbeddingEnv) {
|
|
210
|
+
throw new Error(
|
|
211
|
+
`${removedEmbeddingEnv} is no longer supported. Configure ` +
|
|
212
|
+
"inference.embedding.endpoint with the complete OpenAI-compatible embeddings request URL. " +
|
|
213
|
+
"The old client appended /v1/embeddings.",
|
|
214
|
+
);
|
|
215
|
+
}
|
|
186
216
|
const configPath = resolveConfigPath(path);
|
|
187
217
|
const file = Bun.file(expandHome(configPath));
|
|
188
218
|
|
|
@@ -204,6 +234,27 @@ export async function loadConfig(path?: string): Promise<Config> {
|
|
|
204
234
|
"Legacy inference config is not supported. Move [embedding], [rerank], and remote_timeout_ms under [inference] using config.remote.example.toml.",
|
|
205
235
|
);
|
|
206
236
|
}
|
|
237
|
+
const rawReranker = isPlainObject(parsed.inference)
|
|
238
|
+
? parsed.inference.reranker
|
|
239
|
+
: undefined;
|
|
240
|
+
const rawEmbedding = isPlainObject(parsed.inference)
|
|
241
|
+
? parsed.inference.embedding
|
|
242
|
+
: undefined;
|
|
243
|
+
if (
|
|
244
|
+
isPlainObject(rawReranker) &&
|
|
245
|
+
("provider" in rawReranker || "base_url" in rawReranker)
|
|
246
|
+
) {
|
|
247
|
+
throw new Error(
|
|
248
|
+
"inference.reranker.provider and inference.reranker.base_url are no longer supported. " +
|
|
249
|
+
"Use adapter and the complete endpoint URL instead; the old client appended /rerank.",
|
|
250
|
+
);
|
|
251
|
+
}
|
|
252
|
+
if (isPlainObject(rawEmbedding) && "base_url" in rawEmbedding) {
|
|
253
|
+
throw new Error(
|
|
254
|
+
"inference.embedding.base_url is no longer supported. Use inference.embedding.endpoint " +
|
|
255
|
+
"with the complete OpenAI-compatible embeddings request URL; the old client appended /v1/embeddings.",
|
|
256
|
+
);
|
|
257
|
+
}
|
|
207
258
|
if (isPlainObject(parsed.inference) && parsed.inference.mode === "remote") {
|
|
208
259
|
if (!isPlainObject(parsed.inference.embedding)) {
|
|
209
260
|
throw new Error("Remote inference requires an inference.embedding section.");
|
|
@@ -258,46 +309,76 @@ export async function loadConfig(path?: string): Promise<Config> {
|
|
|
258
309
|
if (merged.inference.embedding?.provider !== "openai") {
|
|
259
310
|
throw new Error('Remote inference.embedding.provider must be "openai".');
|
|
260
311
|
}
|
|
261
|
-
if (merged.inference.reranker && merged.inference.reranker.provider !== "infinity") {
|
|
262
|
-
throw new Error('Remote inference.reranker.provider must be "infinity".');
|
|
263
|
-
}
|
|
264
312
|
if (!Number.isInteger(merged.inference.timeout_ms) || merged.inference.timeout_ms < 100) {
|
|
265
313
|
throw new Error("Remote inference.timeout_ms must be an integer of at least 100.");
|
|
266
314
|
}
|
|
267
|
-
if (!merged.inference.embedding?.
|
|
268
|
-
throw new Error("Remote inference requires inference.embedding
|
|
315
|
+
if (!merged.inference.embedding?.endpoint || !merged.inference.embedding.model || !merged.inference.embedding.dimension) {
|
|
316
|
+
throw new Error("Remote inference requires inference.embedding endpoint, model, and dimension.");
|
|
269
317
|
}
|
|
270
|
-
if (merged.inference.reranker && (!merged.inference.reranker.
|
|
271
|
-
throw new Error("Configured inference.reranker requires
|
|
318
|
+
if (merged.inference.reranker && (!merged.inference.reranker.endpoint || !merged.inference.reranker.model)) {
|
|
319
|
+
throw new Error("Configured inference.reranker requires adapter, endpoint, and model.");
|
|
272
320
|
}
|
|
273
321
|
if (merged.inference.reranker && !merged.inference.thresholds) {
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
const url = new URL(value);
|
|
282
|
-
if (!['http:', 'https:'].includes(url.protocol)) throw new Error();
|
|
283
|
-
} catch {
|
|
284
|
-
throw new Error(`${name} must be an HTTP(S) URL.`);
|
|
322
|
+
const warningKey = "inference.reranker.without-thresholds";
|
|
323
|
+
if (!warnedEnv.has(warningKey)) {
|
|
324
|
+
warnedEnv.add(warningKey);
|
|
325
|
+
console.error(
|
|
326
|
+
"skillmux: configured reranker has no calibrated inference.thresholds; " +
|
|
327
|
+
"routing will remain ambiguous until you run `skillmux calibrate run`.",
|
|
328
|
+
);
|
|
285
329
|
}
|
|
286
330
|
}
|
|
287
|
-
const
|
|
331
|
+
const embedEndpoint = getEnv("SKILLMUX_EMBED_ENDPOINT", "EMBED_ENDPOINT");
|
|
288
332
|
const embedModel = getEnv("SKILLMUX_EMBED_MODEL", "EMBED_MODEL");
|
|
289
333
|
const embedDimStr = getEnv("SKILLMUX_EMBED_DIMENSION", "EMBED_DIMENSION");
|
|
290
|
-
const
|
|
334
|
+
const rerankEndpoint = getEnv("SKILLMUX_RERANK_ENDPOINT", "RERANK_ENDPOINT");
|
|
335
|
+
const rerankAdapter = getEnv("SKILLMUX_RERANK_ADAPTER", "RERANK_ADAPTER");
|
|
291
336
|
const rerankModel = getEnv("SKILLMUX_RERANK_MODEL", "RERANK_MODEL");
|
|
292
|
-
if (
|
|
337
|
+
if (embedEndpoint) merged.inference.embedding.endpoint = embedEndpoint;
|
|
293
338
|
if (embedModel) merged.inference.embedding.model = embedModel;
|
|
294
|
-
if (
|
|
339
|
+
if (rerankEndpoint && merged.inference.reranker) merged.inference.reranker.endpoint = rerankEndpoint;
|
|
340
|
+
if (rerankAdapter && merged.inference.reranker) {
|
|
341
|
+
merged.inference.reranker.adapter = rerankAdapter as "jina-v1" | "bifrost-v1";
|
|
342
|
+
}
|
|
295
343
|
if (rerankModel && merged.inference.reranker) merged.inference.reranker.model = rerankModel;
|
|
296
344
|
if (embedDimStr) {
|
|
297
345
|
const dimension = Number(embedDimStr);
|
|
298
346
|
if (!Number.isInteger(dimension) || dimension < 1) throw new Error(`Invalid embedding dimension: ${embedDimStr}`);
|
|
299
347
|
merged.inference.embedding.dimension = dimension;
|
|
300
348
|
}
|
|
349
|
+
for (const [name, value, exactEndpoint] of [
|
|
350
|
+
["inference.embedding.endpoint", merged.inference.embedding.endpoint, true],
|
|
351
|
+
...(merged.inference.reranker
|
|
352
|
+
? [["inference.reranker.endpoint", merged.inference.reranker.endpoint, true] as const]
|
|
353
|
+
: []),
|
|
354
|
+
] as const) {
|
|
355
|
+
try {
|
|
356
|
+
const url = new URL(value);
|
|
357
|
+
if (!["http:", "https:"].includes(url.protocol)) throw new Error();
|
|
358
|
+
if (exactEndpoint && (url.username || url.password || url.hash)) throw new Error();
|
|
359
|
+
} catch {
|
|
360
|
+
throw new Error(
|
|
361
|
+
exactEndpoint
|
|
362
|
+
? `${name} must be an absolute HTTP(S) URL without userinfo or a fragment.`
|
|
363
|
+
: `${name} must be an HTTP(S) URL.`,
|
|
364
|
+
);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
for (const [name, apiKeyEnv] of [
|
|
368
|
+
["inference.embedding.api_key_env", merged.inference.embedding.api_key_env],
|
|
369
|
+
...(merged.inference.reranker
|
|
370
|
+
? [["inference.reranker.api_key_env", merged.inference.reranker.api_key_env] as const]
|
|
371
|
+
: []),
|
|
372
|
+
] as const) {
|
|
373
|
+
if (
|
|
374
|
+
apiKeyEnv !== undefined &&
|
|
375
|
+
(process.env[apiKeyEnv] === undefined || process.env[apiKeyEnv] === "")
|
|
376
|
+
) {
|
|
377
|
+
throw new Error(
|
|
378
|
+
`${name} names environment variable "${apiKeyEnv}", but it is unset or empty.`,
|
|
379
|
+
);
|
|
380
|
+
}
|
|
381
|
+
}
|
|
301
382
|
} else {
|
|
302
383
|
throw new Error(`Invalid inference.mode: ${(merged.inference as { mode?: unknown }).mode}`);
|
|
303
384
|
}
|
package/src/dataset-generator.ts
CHANGED
|
@@ -13,14 +13,45 @@ export interface GenerateDatasetOptions {
|
|
|
13
13
|
queriesPerSplit?: number;
|
|
14
14
|
}
|
|
15
15
|
|
|
16
|
-
const
|
|
17
|
-
"
|
|
18
|
-
"
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
16
|
+
const STOP_WORDS = new Set([
|
|
17
|
+
"a", "an", "and", "are", "as", "at", "be", "by", "for", "from", "in", "is",
|
|
18
|
+
"it", "of", "on", "or", "the", "this", "to", "use", "with",
|
|
19
|
+
]);
|
|
20
|
+
|
|
21
|
+
function words(value: string): string[] {
|
|
22
|
+
return value
|
|
23
|
+
.toLowerCase()
|
|
24
|
+
.match(/[a-z0-9]+/g)
|
|
25
|
+
?.filter((word) => word.length > 2 && !STOP_WORDS.has(word)) ?? [];
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function anchors(skill: VaultSkill): string[] {
|
|
29
|
+
const preferred = [...skill.aliases.flatMap(words), ...words(skill.title)];
|
|
30
|
+
const fallback = words(skill.description);
|
|
31
|
+
return [...new Set([...preferred, ...fallback])].slice(0, 2);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function matchedQuery(skill: VaultSkill, variant: number): string {
|
|
35
|
+
const [first = "specialized", second = "workflow"] = anchors(skill);
|
|
36
|
+
const templates = [
|
|
37
|
+
`I need practical guidance completing an unfamiliar ${first} ${second} task safely`,
|
|
38
|
+
`Which available workflow can handle my unusual ${first} ${second} problem end to end`,
|
|
39
|
+
`Please guide me through a difficult unfamiliar ${first} ${second} operation safely`,
|
|
40
|
+
];
|
|
41
|
+
return templates[variant % templates.length]!;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function ambiguousQuery(first: VaultSkill, second: VaultSkill): string {
|
|
45
|
+
const [firstAnchor = "first"] = anchors(first);
|
|
46
|
+
const [secondAnchor = "second"] = anchors(second);
|
|
47
|
+
return `Help with a workflow spanning both ${firstAnchor} and ${secondAnchor} responsibilities`;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function nearMissQuery(first: VaultSkill, second: VaultSkill): string {
|
|
51
|
+
const [firstAnchor = "one"] = anchors(first);
|
|
52
|
+
const [secondAnchor = "another"] = anchors(second);
|
|
53
|
+
return `Explain the theory comparing ${firstAnchor} and ${secondAnchor} without performing either workflow`;
|
|
54
|
+
}
|
|
24
55
|
|
|
25
56
|
/**
|
|
26
57
|
* Automatically generate a synthetic decision-policy calibration dataset
|
|
@@ -30,110 +61,58 @@ export function generateDataset(
|
|
|
30
61
|
skills: VaultSkill[],
|
|
31
62
|
options: GenerateDatasetOptions = {},
|
|
32
63
|
): RawDecisionCase[] {
|
|
33
|
-
|
|
64
|
+
if (skills.length < 4) {
|
|
65
|
+
throw new Error(
|
|
66
|
+
"Dataset generation requires at least 4 vault skills so tune and test can each contain matched and ambiguous cases without skill leakage",
|
|
67
|
+
);
|
|
68
|
+
}
|
|
34
69
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
});
|
|
70
|
+
const cases: RawDecisionCase[] = [];
|
|
71
|
+
const sorted = [...skills].sort((a, b) => a.skill_id.localeCompare(b.skill_id));
|
|
72
|
+
const splitAt = Math.ceil(sorted.length / 2);
|
|
73
|
+
const bySplit: Record<DecisionSplit, VaultSkill[]> = {
|
|
74
|
+
tune: sorted.slice(0, splitAt),
|
|
75
|
+
test: sorted.slice(splitAt),
|
|
76
|
+
};
|
|
77
|
+
const targetPerSplit = Math.max(3, options.queriesPerSplit ?? 10);
|
|
44
78
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
split: "test",
|
|
50
|
-
expected_outcome: "matched",
|
|
51
|
-
relevant_skill_ids: [skill.skill_id],
|
|
52
|
-
});
|
|
53
|
-
} else {
|
|
79
|
+
for (const split of ["tune", "test"] as const) {
|
|
80
|
+
const splitSkills = bySplit[split];
|
|
81
|
+
for (let i = 0; i < splitSkills.length; i++) {
|
|
82
|
+
const skill = splitSkills[i]!;
|
|
54
83
|
cases.push({
|
|
55
|
-
query:
|
|
56
|
-
split
|
|
84
|
+
query: matchedQuery(skill, i),
|
|
85
|
+
split,
|
|
57
86
|
expected_outcome: "matched",
|
|
58
87
|
relevant_skill_ids: [skill.skill_id],
|
|
59
88
|
});
|
|
60
89
|
}
|
|
61
|
-
}
|
|
62
90
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
// Pair skills for ambiguous multi-match
|
|
66
|
-
for (let i = 0; i < skills.length - 1; i += 2) {
|
|
67
|
-
const s1 = skills[i]!;
|
|
68
|
-
const s2 = skills[i + 1]!;
|
|
69
|
-
const split: DecisionSplit = i % 4 === 0 ? "tune" : "test";
|
|
70
|
-
cases.push({
|
|
71
|
-
query: `automated task using ${s1.title} and ${s2.title}`,
|
|
72
|
-
split,
|
|
73
|
-
expected_outcome: "ambiguous",
|
|
74
|
-
relevant_skill_ids: [s1.skill_id, s2.skill_id],
|
|
75
|
-
});
|
|
76
|
-
}
|
|
77
|
-
} else {
|
|
78
|
-
// Fallback ambiguous cases if fewer than 2 skills
|
|
79
|
-
cases.push({
|
|
80
|
-
query: "automate browser workflow testing",
|
|
81
|
-
split: "tune",
|
|
82
|
-
expected_outcome: "ambiguous",
|
|
83
|
-
relevant_skill_ids: ["mock-e2e", "mock-browser"],
|
|
84
|
-
});
|
|
85
|
-
cases.push({
|
|
86
|
-
query: "extract and fetch clean web text",
|
|
87
|
-
split: "test",
|
|
88
|
-
expected_outcome: "ambiguous",
|
|
89
|
-
relevant_skill_ids: ["mock-fetch", "mock-extract"],
|
|
90
|
-
});
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
// Ensure both tune and test have ambiguous cases
|
|
94
|
-
if (!cases.some((c) => c.split === "tune" && c.expected_outcome === "ambiguous")) {
|
|
95
|
-
const sIds = skills.length >= 2 ? [skills[0]!.skill_id, skills[1]!.skill_id] : ["mock-a", "mock-b"];
|
|
91
|
+
const first = splitSkills[0]!;
|
|
92
|
+
const second = splitSkills[1]!;
|
|
96
93
|
cases.push({
|
|
97
|
-
query:
|
|
98
|
-
split
|
|
94
|
+
query: ambiguousQuery(first, second),
|
|
95
|
+
split,
|
|
99
96
|
expected_outcome: "ambiguous",
|
|
100
|
-
relevant_skill_ids:
|
|
97
|
+
relevant_skill_ids: [first.skill_id, second.skill_id],
|
|
101
98
|
});
|
|
102
|
-
}
|
|
103
|
-
if (!cases.some((c) => c.split === "test" && c.expected_outcome === "ambiguous")) {
|
|
104
|
-
const sIds = skills.length >= 2 ? [skills[0]!.skill_id, skills[1]!.skill_id] : ["mock-a", "mock-b"];
|
|
105
99
|
cases.push({
|
|
106
|
-
query:
|
|
107
|
-
split
|
|
108
|
-
expected_outcome: "ambiguous",
|
|
109
|
-
relevant_skill_ids: sIds,
|
|
110
|
-
});
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
// --- 3. No Match Cases ---
|
|
114
|
-
GENERIC_NO_MATCH_QUERIES.forEach((q, idx) => {
|
|
115
|
-
cases.push({
|
|
116
|
-
query: q,
|
|
117
|
-
split: idx % 2 === 0 ? "tune" : "test",
|
|
100
|
+
query: nearMissQuery(first, second),
|
|
101
|
+
split,
|
|
118
102
|
expected_outcome: "no_match",
|
|
119
103
|
relevant_skill_ids: [],
|
|
120
104
|
});
|
|
121
|
-
});
|
|
122
105
|
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
split: "test",
|
|
134
|
-
expected_outcome: "matched",
|
|
135
|
-
relevant_skill_ids: ["mock-docs"],
|
|
136
|
-
});
|
|
106
|
+
for (let i = cases.filter((item) => item.split === split).length; i < targetPerSplit; i++) {
|
|
107
|
+
const left = splitSkills[i % splitSkills.length]!;
|
|
108
|
+
const right = splitSkills[(i + 1) % splitSkills.length]!;
|
|
109
|
+
cases.push({
|
|
110
|
+
query: i % 2 === 0 ? matchedQuery(left, i) : nearMissQuery(left, right),
|
|
111
|
+
split,
|
|
112
|
+
expected_outcome: i % 2 === 0 ? "matched" : "no_match",
|
|
113
|
+
relevant_skill_ids: i % 2 === 0 ? [left.skill_id] : [],
|
|
114
|
+
});
|
|
115
|
+
}
|
|
137
116
|
}
|
|
138
117
|
|
|
139
118
|
return cases;
|
package/src/decision.ts
CHANGED
|
@@ -23,7 +23,10 @@ export function decideResolveOutcome({ reranked, candidates, thresholds }: Decis
|
|
|
23
23
|
|| thresholds.match_margin === undefined
|
|
24
24
|
|| thresholds.candidate_floor === undefined
|
|
25
25
|
) {
|
|
26
|
-
|
|
26
|
+
return {
|
|
27
|
+
outcome: "ambiguous",
|
|
28
|
+
candidates: candidates.slice(0, thresholds.candidate_limit),
|
|
29
|
+
};
|
|
27
30
|
}
|
|
28
31
|
const { match_score, match_margin, candidate_floor } = thresholds;
|
|
29
32
|
|
package/src/doctor.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { existsSync, mkdirSync } from "node:fs";
|
|
2
|
-
import { createClients } from "./clients";
|
|
2
|
+
import { createClients, RemoteInferenceError } from "./clients";
|
|
3
3
|
import { embeddingDimension, expandHome } from "./config";
|
|
4
4
|
import { parseManifest, resolveManifestPath, validateManifest } from "./manifest";
|
|
5
5
|
import { readSkillmuxMarker } from "./sync";
|
|
@@ -10,6 +10,7 @@ export interface DoctorCheck {
|
|
|
10
10
|
name: string;
|
|
11
11
|
ok: boolean;
|
|
12
12
|
detail: string;
|
|
13
|
+
failure_kind?: "configuration" | "availability" | "protocol" | "unexpected";
|
|
13
14
|
}
|
|
14
15
|
|
|
15
16
|
export interface DoctorReport {
|
|
@@ -99,8 +100,13 @@ export async function diagnose(config: Config): Promise<DoctorReport> {
|
|
|
99
100
|
}
|
|
100
101
|
}
|
|
101
102
|
|
|
103
|
+
const inferenceFailure = (error: unknown): Pick<DoctorCheck, "detail" | "failure_kind"> =>
|
|
104
|
+
error instanceof RemoteInferenceError
|
|
105
|
+
? { detail: error.message, failure_kind: error.kind }
|
|
106
|
+
: { detail: "unexpected inference failure", failure_kind: "unexpected" };
|
|
107
|
+
|
|
108
|
+
const clients = createClients(config);
|
|
102
109
|
try {
|
|
103
|
-
const clients = createClients(config);
|
|
104
110
|
const vectors = await clients.embed(["skill router diagnostic"]);
|
|
105
111
|
const actualDimension = vectors[0]?.length ?? 0;
|
|
106
112
|
checks.push({
|
|
@@ -108,14 +114,19 @@ export async function diagnose(config: Config): Promise<DoctorReport> {
|
|
|
108
114
|
ok: actualDimension === embeddingDimension(config),
|
|
109
115
|
detail: `dimension ${actualDimension}`,
|
|
110
116
|
});
|
|
111
|
-
|
|
117
|
+
} catch (error) {
|
|
118
|
+
checks.push({ name: "embedding", ok: false, ...inferenceFailure(error) });
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
if (clients.rerank) {
|
|
122
|
+
try {
|
|
112
123
|
const scores = await clients.rerank("skill router diagnostic", [
|
|
113
124
|
{ skill_id: "doctor", text: "Routes a task to an appropriate skill." },
|
|
114
125
|
]);
|
|
115
126
|
checks.push({ name: "reranker", ok: scores.length === 1 && Number.isFinite(scores[0]), detail: "one finite score" });
|
|
127
|
+
} catch (error) {
|
|
128
|
+
checks.push({ name: "reranker", ok: false, ...inferenceFailure(error) });
|
|
116
129
|
}
|
|
117
|
-
} catch (error) {
|
|
118
|
-
checks.push({ name: "inference", ok: false, detail: String(error) });
|
|
119
130
|
}
|
|
120
131
|
|
|
121
132
|
const inferenceReady = checks.some((check) => check.name === "embedding" && check.ok);
|
package/src/eval.ts
CHANGED
|
@@ -78,7 +78,8 @@ export async function evalVault(cases = loadEvalCases()): Promise<EvalReport> {
|
|
|
78
78
|
const hybridRankings: string[][] = [];
|
|
79
79
|
for (const evalCase of cases) {
|
|
80
80
|
const lexical = ftsSearch(db, evalCase.query, config.recall.k_lexical);
|
|
81
|
-
const vector = (await clients.embed([evalCase.query]))[0]
|
|
81
|
+
const vector = (await clients.embed([evalCase.query]))[0];
|
|
82
|
+
if (!vector) throw new Error("Embedding client returned no query vector.");
|
|
82
83
|
const semantic = vectorTopK(db, vector, config.recall.k_vector);
|
|
83
84
|
lexicalRankings.push(lexical.map((row) => row.skill_id));
|
|
84
85
|
hybridRankings.push(reciprocalRankFusion<SkillRow>(lexical, semantic).map((row) => row.skill_id));
|
package/src/router-core.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { watch } from "node:fs";
|
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import { buildAuditRow } from "./audit";
|
|
5
5
|
import { embeddingDimension, embeddingFingerprint, expandHome, loadConfig } from "./config";
|
|
6
|
+
import { RemoteInferenceError } from "./clients";
|
|
6
7
|
import {
|
|
7
8
|
deleteSkill,
|
|
8
9
|
findExactMatch,
|
|
@@ -269,12 +270,22 @@ export async function backfillEmbeddings(): Promise<number> {
|
|
|
269
270
|
const chunk = pending.slice(i, i + BATCH_SIZE);
|
|
270
271
|
try {
|
|
271
272
|
const vectors = await clients.embed(chunk.map(rerankText));
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
273
|
+
db.transaction(() => {
|
|
274
|
+
chunk.forEach((row, j) => {
|
|
275
|
+
const vector = vectors[j];
|
|
276
|
+
if (!vector) throw new Error("Embedding client returned an incomplete batch.");
|
|
277
|
+
upsertVector(db, row.skill_id, row.content_sha256, fingerprint, vector);
|
|
278
|
+
});
|
|
279
|
+
})();
|
|
275
280
|
count += chunk.length;
|
|
276
281
|
} catch (err) {
|
|
277
|
-
if (
|
|
282
|
+
if (
|
|
283
|
+
i === 0 ||
|
|
284
|
+
(err instanceof RemoteInferenceError &&
|
|
285
|
+
(err.kind === "configuration" || err.kind === "protocol"))
|
|
286
|
+
) {
|
|
287
|
+
throw err;
|
|
288
|
+
}
|
|
278
289
|
break;
|
|
279
290
|
}
|
|
280
291
|
}
|
|
@@ -384,44 +395,7 @@ export async function resolveSkill(input: ResolveSkillInput): Promise<ResolveRes
|
|
|
384
395
|
return result;
|
|
385
396
|
}
|
|
386
397
|
|
|
387
|
-
const
|
|
388
|
-
|
|
389
|
-
const lexical = ftsSearch(db, input.query, config.recall.k_lexical);
|
|
390
|
-
|
|
391
|
-
let retrieval: RetrievalCapability = "lexical";
|
|
392
|
-
let rows = lexical;
|
|
393
|
-
if (!input.forceLexical) {
|
|
394
|
-
try {
|
|
395
|
-
const queryVec = (await clients.embed([input.query]))[0]!;
|
|
396
|
-
const nearest = vectorTopK(db, queryVec, config.recall.k_vector);
|
|
397
|
-
rows = reciprocalRankFusion(lexical, nearest);
|
|
398
|
-
retrieval = "hybrid";
|
|
399
|
-
} catch {
|
|
400
|
-
retrieval = "lexical";
|
|
401
|
-
}
|
|
402
|
-
}
|
|
403
|
-
|
|
404
|
-
let scores: number[] | null = null;
|
|
405
|
-
if (clients.rerank && retrieval === "hybrid" && rows.length > 0) {
|
|
406
|
-
try {
|
|
407
|
-
scores = await clients.rerank(
|
|
408
|
-
input.query,
|
|
409
|
-
rows.map((r) => ({ skill_id: r.skill_id, text: rerankText(r) })),
|
|
410
|
-
);
|
|
411
|
-
retrieval = "reranked";
|
|
412
|
-
} catch {
|
|
413
|
-
scores = null;
|
|
414
|
-
}
|
|
415
|
-
}
|
|
416
|
-
|
|
417
|
-
const rankedCandidates: RankedCandidate[] = rows
|
|
418
|
-
.map((r, i) => ({
|
|
419
|
-
skill_id: r.skill_id,
|
|
420
|
-
title: r.title,
|
|
421
|
-
description: r.description,
|
|
422
|
-
score: scores?.[i] ?? null,
|
|
423
|
-
}))
|
|
424
|
-
.sort((a, b) => scores === null ? 0 : (b.score ?? -Infinity) - (a.score ?? -Infinity));
|
|
398
|
+
const { retrieval, candidates: rankedCandidates } = await retrieveAndRerank(input);
|
|
425
399
|
|
|
426
400
|
const decisionThresholds = retrieval === "reranked" && config.inference.mode === "remote"
|
|
427
401
|
? { candidate_limit: config.thresholds.candidate_limit, ...config.inference.thresholds }
|
|
@@ -473,6 +447,63 @@ export async function resolveSkill(input: ResolveSkillInput): Promise<ResolveRes
|
|
|
473
447
|
return result;
|
|
474
448
|
}
|
|
475
449
|
|
|
450
|
+
export interface RetrievalResult {
|
|
451
|
+
retrieval: Exclude<RetrievalCapability, "exact">;
|
|
452
|
+
candidates: RankedCandidate[];
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
/**
|
|
456
|
+
* Retrieve the full fused candidate set and rerank it once without applying
|
|
457
|
+
* decision thresholds. Calibration uses this to avoid observing the policy it
|
|
458
|
+
* is trying to replace.
|
|
459
|
+
*/
|
|
460
|
+
export async function retrieveAndRerank(
|
|
461
|
+
input: ResolveSkillInput,
|
|
462
|
+
): Promise<RetrievalResult> {
|
|
463
|
+
const { config, db } = await getEnv();
|
|
464
|
+
await syncVaultIfNeeded();
|
|
465
|
+
const clients = getClients();
|
|
466
|
+
const lexical = ftsSearch(db, input.query, config.recall.k_lexical);
|
|
467
|
+
|
|
468
|
+
let retrieval: RetrievalResult["retrieval"] = "lexical";
|
|
469
|
+
let rows = lexical;
|
|
470
|
+
if (!input.forceLexical) {
|
|
471
|
+
try {
|
|
472
|
+
const queryVec = (await clients.embed([input.query]))[0];
|
|
473
|
+
if (!queryVec) throw new Error("Embedding client returned no query vector.");
|
|
474
|
+
const nearest = vectorTopK(db, queryVec, config.recall.k_vector);
|
|
475
|
+
rows = reciprocalRankFusion(lexical, nearest);
|
|
476
|
+
retrieval = "hybrid";
|
|
477
|
+
} catch {
|
|
478
|
+
retrieval = "lexical";
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
let scores: number[] | null = null;
|
|
483
|
+
if (clients.rerank && retrieval === "hybrid" && rows.length > 0) {
|
|
484
|
+
try {
|
|
485
|
+
scores = await clients.rerank(
|
|
486
|
+
input.query,
|
|
487
|
+
rows.map((r) => ({ skill_id: r.skill_id, text: rerankText(r) })),
|
|
488
|
+
);
|
|
489
|
+
retrieval = "reranked";
|
|
490
|
+
} catch {
|
|
491
|
+
scores = null;
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
const candidates = rows
|
|
496
|
+
.map((r, i) => ({
|
|
497
|
+
skill_id: r.skill_id,
|
|
498
|
+
title: r.title,
|
|
499
|
+
description: r.description,
|
|
500
|
+
score: scores?.[i] ?? null,
|
|
501
|
+
}))
|
|
502
|
+
.sort((a, b) => scores === null ? 0 : (b.score ?? -Infinity) - (a.score ?? -Infinity));
|
|
503
|
+
|
|
504
|
+
return { retrieval, candidates };
|
|
505
|
+
}
|
|
506
|
+
|
|
476
507
|
export async function fetchSkill(input: FetchSkillInput): Promise<FetchSkillResult> {
|
|
477
508
|
const { config, db } = await getEnv();
|
|
478
509
|
await syncVaultIfNeeded();
|