@klhapp/skillmux 1.0.1 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/config.ts CHANGED
@@ -48,14 +48,14 @@ const configSchema = z.object({
48
48
  timeout_ms: z.number().int().min(100),
49
49
  embedding: z.object({
50
50
  provider: z.literal("openai"),
51
- base_url: z.url(),
51
+ endpoint: z.url(),
52
52
  model: z.string().min(1),
53
53
  dimension: z.number().int().positive(),
54
54
  api_key_env: z.string().min(1).optional(),
55
55
  }).strict(),
56
56
  reranker: z.object({
57
- provider: z.literal("infinity"),
58
- base_url: z.url(),
57
+ adapter: z.enum(["jina-v1", "bifrost-v1"]),
58
+ endpoint: z.url(),
59
59
  model: z.string().min(1),
60
60
  api_key_env: z.string().min(1).optional(),
61
61
  }).strict().optional(),
@@ -133,6 +133,12 @@ export function embeddingFingerprint(config: Config): string {
133
133
  return `${implementation}:${inference.embedding.model}:${inference.embedding.dimension}`;
134
134
  }
135
135
 
136
+ export function rerankerFingerprint(config: Config): string | undefined {
137
+ const inference = config.inference;
138
+ if (inference.mode !== "remote" || !inference.reranker) return undefined;
139
+ return `remote:${inference.reranker.adapter}:${inference.reranker.model}`;
140
+ }
141
+
136
142
  export function expandHome(path: string): string {
137
143
  return path.startsWith("~") ? join(homedir(), path.slice(1)) : path;
138
144
  }
@@ -183,6 +189,30 @@ export function resolveConfigPath(path?: string): string {
183
189
 
184
190
  export async function loadConfig(path?: string): Promise<Config> {
185
191
  migrateLegacyPaths();
192
+ const removedRerankerEnv = [
193
+ "SKILLMUX_RERANK_BASE_URL",
194
+ "SKILL_ROUTER_RERANK_BASE_URL",
195
+ "RERANK_BASE_URL",
196
+ ].find((name) => process.env[name] !== undefined);
197
+ if (removedRerankerEnv) {
198
+ throw new Error(
199
+ `${removedRerankerEnv} is no longer supported. Configure ` +
200
+ "inference.reranker.endpoint with the complete request URL and " +
201
+ 'inference.reranker.adapter (for example, "jina-v1"). The old client appended /rerank.',
202
+ );
203
+ }
204
+ const removedEmbeddingEnv = [
205
+ "SKILLMUX_EMBED_BASE_URL",
206
+ "SKILL_ROUTER_EMBED_BASE_URL",
207
+ "EMBED_BASE_URL",
208
+ ].find((name) => process.env[name] !== undefined);
209
+ if (removedEmbeddingEnv) {
210
+ throw new Error(
211
+ `${removedEmbeddingEnv} is no longer supported. Configure ` +
212
+ "inference.embedding.endpoint with the complete OpenAI-compatible embeddings request URL. " +
213
+ "The old client appended /v1/embeddings.",
214
+ );
215
+ }
186
216
  const configPath = resolveConfigPath(path);
187
217
  const file = Bun.file(expandHome(configPath));
188
218
 
@@ -204,6 +234,27 @@ export async function loadConfig(path?: string): Promise<Config> {
204
234
  "Legacy inference config is not supported. Move [embedding], [rerank], and remote_timeout_ms under [inference] using config.remote.example.toml.",
205
235
  );
206
236
  }
237
+ const rawReranker = isPlainObject(parsed.inference)
238
+ ? parsed.inference.reranker
239
+ : undefined;
240
+ const rawEmbedding = isPlainObject(parsed.inference)
241
+ ? parsed.inference.embedding
242
+ : undefined;
243
+ if (
244
+ isPlainObject(rawReranker) &&
245
+ ("provider" in rawReranker || "base_url" in rawReranker)
246
+ ) {
247
+ throw new Error(
248
+ "inference.reranker.provider and inference.reranker.base_url are no longer supported. " +
249
+ "Use adapter and the complete endpoint URL instead; the old client appended /rerank.",
250
+ );
251
+ }
252
+ if (isPlainObject(rawEmbedding) && "base_url" in rawEmbedding) {
253
+ throw new Error(
254
+ "inference.embedding.base_url is no longer supported. Use inference.embedding.endpoint " +
255
+ "with the complete OpenAI-compatible embeddings request URL; the old client appended /v1/embeddings.",
256
+ );
257
+ }
207
258
  if (isPlainObject(parsed.inference) && parsed.inference.mode === "remote") {
208
259
  if (!isPlainObject(parsed.inference.embedding)) {
209
260
  throw new Error("Remote inference requires an inference.embedding section.");
@@ -258,46 +309,76 @@ export async function loadConfig(path?: string): Promise<Config> {
258
309
  if (merged.inference.embedding?.provider !== "openai") {
259
310
  throw new Error('Remote inference.embedding.provider must be "openai".');
260
311
  }
261
- if (merged.inference.reranker && merged.inference.reranker.provider !== "infinity") {
262
- throw new Error('Remote inference.reranker.provider must be "infinity".');
263
- }
264
312
  if (!Number.isInteger(merged.inference.timeout_ms) || merged.inference.timeout_ms < 100) {
265
313
  throw new Error("Remote inference.timeout_ms must be an integer of at least 100.");
266
314
  }
267
- if (!merged.inference.embedding?.base_url || !merged.inference.embedding.model || !merged.inference.embedding.dimension) {
268
- throw new Error("Remote inference requires inference.embedding base_url, model, and dimension.");
315
+ if (!merged.inference.embedding?.endpoint || !merged.inference.embedding.model || !merged.inference.embedding.dimension) {
316
+ throw new Error("Remote inference requires inference.embedding endpoint, model, and dimension.");
269
317
  }
270
- if (merged.inference.reranker && (!merged.inference.reranker.base_url || !merged.inference.reranker.model)) {
271
- throw new Error("Configured inference.reranker requires base_url and model.");
318
+ if (merged.inference.reranker && (!merged.inference.reranker.endpoint || !merged.inference.reranker.model)) {
319
+ throw new Error("Configured inference.reranker requires adapter, endpoint, and model.");
272
320
  }
273
321
  if (merged.inference.reranker && !merged.inference.thresholds) {
274
- throw new Error("Configured inference.reranker requires calibrated inference.thresholds.");
275
- }
276
- for (const [name, value] of [
277
- ["inference.embedding.base_url", merged.inference.embedding.base_url],
278
- ...(merged.inference.reranker ? [["inference.reranker.base_url", merged.inference.reranker.base_url] as const] : []),
279
- ] as const) {
280
- try {
281
- const url = new URL(value);
282
- if (!['http:', 'https:'].includes(url.protocol)) throw new Error();
283
- } catch {
284
- throw new Error(`${name} must be an HTTP(S) URL.`);
322
+ const warningKey = "inference.reranker.without-thresholds";
323
+ if (!warnedEnv.has(warningKey)) {
324
+ warnedEnv.add(warningKey);
325
+ console.error(
326
+ "skillmux: configured reranker has no calibrated inference.thresholds; " +
327
+ "routing will remain ambiguous until you run `skillmux calibrate run`.",
328
+ );
285
329
  }
286
330
  }
287
- const embedUrl = getEnv("SKILLMUX_EMBED_BASE_URL", "EMBED_BASE_URL");
331
+ const embedEndpoint = getEnv("SKILLMUX_EMBED_ENDPOINT", "EMBED_ENDPOINT");
288
332
  const embedModel = getEnv("SKILLMUX_EMBED_MODEL", "EMBED_MODEL");
289
333
  const embedDimStr = getEnv("SKILLMUX_EMBED_DIMENSION", "EMBED_DIMENSION");
290
- const rerankUrl = getEnv("SKILLMUX_RERANK_BASE_URL", "RERANK_BASE_URL");
334
+ const rerankEndpoint = getEnv("SKILLMUX_RERANK_ENDPOINT", "RERANK_ENDPOINT");
335
+ const rerankAdapter = getEnv("SKILLMUX_RERANK_ADAPTER", "RERANK_ADAPTER");
291
336
  const rerankModel = getEnv("SKILLMUX_RERANK_MODEL", "RERANK_MODEL");
292
- if (embedUrl) merged.inference.embedding.base_url = embedUrl;
337
+ if (embedEndpoint) merged.inference.embedding.endpoint = embedEndpoint;
293
338
  if (embedModel) merged.inference.embedding.model = embedModel;
294
- if (rerankUrl && merged.inference.reranker) merged.inference.reranker.base_url = rerankUrl;
339
+ if (rerankEndpoint && merged.inference.reranker) merged.inference.reranker.endpoint = rerankEndpoint;
340
+ if (rerankAdapter && merged.inference.reranker) {
341
+ merged.inference.reranker.adapter = rerankAdapter as "jina-v1" | "bifrost-v1";
342
+ }
295
343
  if (rerankModel && merged.inference.reranker) merged.inference.reranker.model = rerankModel;
296
344
  if (embedDimStr) {
297
345
  const dimension = Number(embedDimStr);
298
346
  if (!Number.isInteger(dimension) || dimension < 1) throw new Error(`Invalid embedding dimension: ${embedDimStr}`);
299
347
  merged.inference.embedding.dimension = dimension;
300
348
  }
349
+ for (const [name, value, exactEndpoint] of [
350
+ ["inference.embedding.endpoint", merged.inference.embedding.endpoint, true],
351
+ ...(merged.inference.reranker
352
+ ? [["inference.reranker.endpoint", merged.inference.reranker.endpoint, true] as const]
353
+ : []),
354
+ ] as const) {
355
+ try {
356
+ const url = new URL(value);
357
+ if (!["http:", "https:"].includes(url.protocol)) throw new Error();
358
+ if (exactEndpoint && (url.username || url.password || url.hash)) throw new Error();
359
+ } catch {
360
+ throw new Error(
361
+ exactEndpoint
362
+ ? `${name} must be an absolute HTTP(S) URL without userinfo or a fragment.`
363
+ : `${name} must be an HTTP(S) URL.`,
364
+ );
365
+ }
366
+ }
367
+ for (const [name, apiKeyEnv] of [
368
+ ["inference.embedding.api_key_env", merged.inference.embedding.api_key_env],
369
+ ...(merged.inference.reranker
370
+ ? [["inference.reranker.api_key_env", merged.inference.reranker.api_key_env] as const]
371
+ : []),
372
+ ] as const) {
373
+ if (
374
+ apiKeyEnv !== undefined &&
375
+ (process.env[apiKeyEnv] === undefined || process.env[apiKeyEnv] === "")
376
+ ) {
377
+ throw new Error(
378
+ `${name} names environment variable "${apiKeyEnv}", but it is unset or empty.`,
379
+ );
380
+ }
381
+ }
301
382
  } else {
302
383
  throw new Error(`Invalid inference.mode: ${(merged.inference as { mode?: unknown }).mode}`);
303
384
  }
@@ -13,14 +13,45 @@ export interface GenerateDatasetOptions {
13
13
  queriesPerSplit?: number;
14
14
  }
15
15
 
16
- const GENERIC_NO_MATCH_QUERIES = [
17
- "what is the weather in Paris today",
18
- "recipe for baking sourdough bread at home",
19
- "what is the distance from Earth to Mars",
20
- "explain quantum entanglement simply",
21
- "how do I solve a quadratic equation",
22
- "who won the 1998 World Cup",
23
- ];
16
+ const STOP_WORDS = new Set([
17
+ "a", "an", "and", "are", "as", "at", "be", "by", "for", "from", "in", "is",
18
+ "it", "of", "on", "or", "the", "this", "to", "use", "with",
19
+ ]);
20
+
21
+ function words(value: string): string[] {
22
+ return value
23
+ .toLowerCase()
24
+ .match(/[a-z0-9]+/g)
25
+ ?.filter((word) => word.length > 2 && !STOP_WORDS.has(word)) ?? [];
26
+ }
27
+
28
+ function anchors(skill: VaultSkill): string[] {
29
+ const preferred = [...skill.aliases.flatMap(words), ...words(skill.title)];
30
+ const fallback = words(skill.description);
31
+ return [...new Set([...preferred, ...fallback])].slice(0, 2);
32
+ }
33
+
34
+ function matchedQuery(skill: VaultSkill, variant: number): string {
35
+ const [first = "specialized", second = "workflow"] = anchors(skill);
36
+ const templates = [
37
+ `I need practical guidance completing an unfamiliar ${first} ${second} task safely`,
38
+ `Which available workflow can handle my unusual ${first} ${second} problem end to end`,
39
+ `Please guide me through a difficult unfamiliar ${first} ${second} operation safely`,
40
+ ];
41
+ return templates[variant % templates.length]!;
42
+ }
43
+
44
+ function ambiguousQuery(first: VaultSkill, second: VaultSkill): string {
45
+ const [firstAnchor = "first"] = anchors(first);
46
+ const [secondAnchor = "second"] = anchors(second);
47
+ return `Help with a workflow spanning both ${firstAnchor} and ${secondAnchor} responsibilities`;
48
+ }
49
+
50
+ function nearMissQuery(first: VaultSkill, second: VaultSkill): string {
51
+ const [firstAnchor = "one"] = anchors(first);
52
+ const [secondAnchor = "another"] = anchors(second);
53
+ return `Explain the theory comparing ${firstAnchor} and ${secondAnchor} without performing either workflow`;
54
+ }
24
55
 
25
56
  /**
26
57
  * Automatically generate a synthetic decision-policy calibration dataset
@@ -30,110 +61,58 @@ export function generateDataset(
30
61
  skills: VaultSkill[],
31
62
  options: GenerateDatasetOptions = {},
32
63
  ): RawDecisionCase[] {
33
- const cases: RawDecisionCase[] = [];
64
+ if (skills.length < 4) {
65
+ throw new Error(
66
+ "Dataset generation requires at least 4 vault skills so tune and test can each contain matched and ambiguous cases without skill leakage",
67
+ );
68
+ }
34
69
 
35
- // --- 1. Matched Cases ---
36
- for (const skill of skills) {
37
- // Primary query from title + description
38
- cases.push({
39
- query: `how do I ${skill.title.toLowerCase()}: ${skill.description.toLowerCase()}`,
40
- split: "tune",
41
- expected_outcome: "matched",
42
- relevant_skill_ids: [skill.skill_id],
43
- });
70
+ const cases: RawDecisionCase[] = [];
71
+ const sorted = [...skills].sort((a, b) => a.skill_id.localeCompare(b.skill_id));
72
+ const splitAt = Math.ceil(sorted.length / 2);
73
+ const bySplit: Record<DecisionSplit, VaultSkill[]> = {
74
+ tune: sorted.slice(0, splitAt),
75
+ test: sorted.slice(splitAt),
76
+ };
77
+ const targetPerSplit = Math.max(3, options.queriesPerSplit ?? 10);
44
78
 
45
- // Secondary query from aliases
46
- if (skill.aliases.length > 0) {
47
- cases.push({
48
- query: `help me with ${skill.aliases[0]}`,
49
- split: "test",
50
- expected_outcome: "matched",
51
- relevant_skill_ids: [skill.skill_id],
52
- });
53
- } else {
79
+ for (const split of ["tune", "test"] as const) {
80
+ const splitSkills = bySplit[split];
81
+ for (let i = 0; i < splitSkills.length; i++) {
82
+ const skill = splitSkills[i]!;
54
83
  cases.push({
55
- query: `execute task related to ${skill.title}`,
56
- split: "test",
84
+ query: matchedQuery(skill, i),
85
+ split,
57
86
  expected_outcome: "matched",
58
87
  relevant_skill_ids: [skill.skill_id],
59
88
  });
60
89
  }
61
- }
62
90
 
63
- // --- 2. Ambiguous Cases ---
64
- if (skills.length >= 2) {
65
- // Pair skills for ambiguous multi-match
66
- for (let i = 0; i < skills.length - 1; i += 2) {
67
- const s1 = skills[i]!;
68
- const s2 = skills[i + 1]!;
69
- const split: DecisionSplit = i % 4 === 0 ? "tune" : "test";
70
- cases.push({
71
- query: `automated task using ${s1.title} and ${s2.title}`,
72
- split,
73
- expected_outcome: "ambiguous",
74
- relevant_skill_ids: [s1.skill_id, s2.skill_id],
75
- });
76
- }
77
- } else {
78
- // Fallback ambiguous cases if fewer than 2 skills
79
- cases.push({
80
- query: "automate browser workflow testing",
81
- split: "tune",
82
- expected_outcome: "ambiguous",
83
- relevant_skill_ids: ["mock-e2e", "mock-browser"],
84
- });
85
- cases.push({
86
- query: "extract and fetch clean web text",
87
- split: "test",
88
- expected_outcome: "ambiguous",
89
- relevant_skill_ids: ["mock-fetch", "mock-extract"],
90
- });
91
- }
92
-
93
- // Ensure both tune and test have ambiguous cases
94
- if (!cases.some((c) => c.split === "tune" && c.expected_outcome === "ambiguous")) {
95
- const sIds = skills.length >= 2 ? [skills[0]!.skill_id, skills[1]!.skill_id] : ["mock-a", "mock-b"];
91
+ const first = splitSkills[0]!;
92
+ const second = splitSkills[1]!;
96
93
  cases.push({
97
- query: "integrated workflow multi skill query",
98
- split: "tune",
94
+ query: ambiguousQuery(first, second),
95
+ split,
99
96
  expected_outcome: "ambiguous",
100
- relevant_skill_ids: sIds,
97
+ relevant_skill_ids: [first.skill_id, second.skill_id],
101
98
  });
102
- }
103
- if (!cases.some((c) => c.split === "test" && c.expected_outcome === "ambiguous")) {
104
- const sIds = skills.length >= 2 ? [skills[0]!.skill_id, skills[1]!.skill_id] : ["mock-a", "mock-b"];
105
99
  cases.push({
106
- query: "combined operations multi skill query",
107
- split: "test",
108
- expected_outcome: "ambiguous",
109
- relevant_skill_ids: sIds,
110
- });
111
- }
112
-
113
- // --- 3. No Match Cases ---
114
- GENERIC_NO_MATCH_QUERIES.forEach((q, idx) => {
115
- cases.push({
116
- query: q,
117
- split: idx % 2 === 0 ? "tune" : "test",
100
+ query: nearMissQuery(first, second),
101
+ split,
118
102
  expected_outcome: "no_match",
119
103
  relevant_skill_ids: [],
120
104
  });
121
- });
122
105
 
123
- // Ensure both splits have at least 1 matched case if skills were empty
124
- if (skills.length === 0) {
125
- cases.push({
126
- query: "run mock container action",
127
- split: "tune",
128
- expected_outcome: "matched",
129
- relevant_skill_ids: ["mock-container"],
130
- });
131
- cases.push({
132
- query: "search mock API docs",
133
- split: "test",
134
- expected_outcome: "matched",
135
- relevant_skill_ids: ["mock-docs"],
136
- });
106
+ for (let i = cases.filter((item) => item.split === split).length; i < targetPerSplit; i++) {
107
+ const left = splitSkills[i % splitSkills.length]!;
108
+ const right = splitSkills[(i + 1) % splitSkills.length]!;
109
+ cases.push({
110
+ query: i % 2 === 0 ? matchedQuery(left, i) : nearMissQuery(left, right),
111
+ split,
112
+ expected_outcome: i % 2 === 0 ? "matched" : "no_match",
113
+ relevant_skill_ids: i % 2 === 0 ? [left.skill_id] : [],
114
+ });
115
+ }
137
116
  }
138
117
 
139
118
  return cases;
package/src/decision.ts CHANGED
@@ -23,7 +23,10 @@ export function decideResolveOutcome({ reranked, candidates, thresholds }: Decis
23
23
  || thresholds.match_margin === undefined
24
24
  || thresholds.candidate_floor === undefined
25
25
  ) {
26
- throw new Error("Reranked decisions require calibrated thresholds.");
26
+ return {
27
+ outcome: "ambiguous",
28
+ candidates: candidates.slice(0, thresholds.candidate_limit),
29
+ };
27
30
  }
28
31
  const { match_score, match_margin, candidate_floor } = thresholds;
29
32
 
package/src/doctor.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import { existsSync, mkdirSync } from "node:fs";
2
- import { createClients } from "./clients";
2
+ import { createClients, RemoteInferenceError } from "./clients";
3
3
  import { embeddingDimension, expandHome } from "./config";
4
4
  import { parseManifest, resolveManifestPath, validateManifest } from "./manifest";
5
5
  import { readSkillmuxMarker } from "./sync";
@@ -10,6 +10,7 @@ export interface DoctorCheck {
10
10
  name: string;
11
11
  ok: boolean;
12
12
  detail: string;
13
+ failure_kind?: "configuration" | "availability" | "protocol" | "unexpected";
13
14
  }
14
15
 
15
16
  export interface DoctorReport {
@@ -99,8 +100,13 @@ export async function diagnose(config: Config): Promise<DoctorReport> {
99
100
  }
100
101
  }
101
102
 
103
+ const inferenceFailure = (error: unknown): Pick<DoctorCheck, "detail" | "failure_kind"> =>
104
+ error instanceof RemoteInferenceError
105
+ ? { detail: error.message, failure_kind: error.kind }
106
+ : { detail: "unexpected inference failure", failure_kind: "unexpected" };
107
+
108
+ const clients = createClients(config);
102
109
  try {
103
- const clients = createClients(config);
104
110
  const vectors = await clients.embed(["skill router diagnostic"]);
105
111
  const actualDimension = vectors[0]?.length ?? 0;
106
112
  checks.push({
@@ -108,14 +114,19 @@ export async function diagnose(config: Config): Promise<DoctorReport> {
108
114
  ok: actualDimension === embeddingDimension(config),
109
115
  detail: `dimension ${actualDimension}`,
110
116
  });
111
- if (clients.rerank) {
117
+ } catch (error) {
118
+ checks.push({ name: "embedding", ok: false, ...inferenceFailure(error) });
119
+ }
120
+
121
+ if (clients.rerank) {
122
+ try {
112
123
  const scores = await clients.rerank("skill router diagnostic", [
113
124
  { skill_id: "doctor", text: "Routes a task to an appropriate skill." },
114
125
  ]);
115
126
  checks.push({ name: "reranker", ok: scores.length === 1 && Number.isFinite(scores[0]), detail: "one finite score" });
127
+ } catch (error) {
128
+ checks.push({ name: "reranker", ok: false, ...inferenceFailure(error) });
116
129
  }
117
- } catch (error) {
118
- checks.push({ name: "inference", ok: false, detail: String(error) });
119
130
  }
120
131
 
121
132
  const inferenceReady = checks.some((check) => check.name === "embedding" && check.ok);
package/src/eval.ts CHANGED
@@ -78,7 +78,8 @@ export async function evalVault(cases = loadEvalCases()): Promise<EvalReport> {
78
78
  const hybridRankings: string[][] = [];
79
79
  for (const evalCase of cases) {
80
80
  const lexical = ftsSearch(db, evalCase.query, config.recall.k_lexical);
81
- const vector = (await clients.embed([evalCase.query]))[0]!;
81
+ const vector = (await clients.embed([evalCase.query]))[0];
82
+ if (!vector) throw new Error("Embedding client returned no query vector.");
82
83
  const semantic = vectorTopK(db, vector, config.recall.k_vector);
83
84
  lexicalRankings.push(lexical.map((row) => row.skill_id));
84
85
  hybridRankings.push(reciprocalRankFusion<SkillRow>(lexical, semantic).map((row) => row.skill_id));
@@ -3,6 +3,7 @@ import { watch } from "node:fs";
3
3
  import { join } from "node:path";
4
4
  import { buildAuditRow } from "./audit";
5
5
  import { embeddingDimension, embeddingFingerprint, expandHome, loadConfig } from "./config";
6
+ import { RemoteInferenceError } from "./clients";
6
7
  import {
7
8
  deleteSkill,
8
9
  findExactMatch,
@@ -269,12 +270,22 @@ export async function backfillEmbeddings(): Promise<number> {
269
270
  const chunk = pending.slice(i, i + BATCH_SIZE);
270
271
  try {
271
272
  const vectors = await clients.embed(chunk.map(rerankText));
272
- chunk.forEach((row, j) => {
273
- upsertVector(db, row.skill_id, row.content_sha256, fingerprint, vectors[j]!);
274
- });
273
+ db.transaction(() => {
274
+ chunk.forEach((row, j) => {
275
+ const vector = vectors[j];
276
+ if (!vector) throw new Error("Embedding client returned an incomplete batch.");
277
+ upsertVector(db, row.skill_id, row.content_sha256, fingerprint, vector);
278
+ });
279
+ })();
275
280
  count += chunk.length;
276
281
  } catch (err) {
277
- if (i === 0) throw err;
282
+ if (
283
+ i === 0 ||
284
+ (err instanceof RemoteInferenceError &&
285
+ (err.kind === "configuration" || err.kind === "protocol"))
286
+ ) {
287
+ throw err;
288
+ }
278
289
  break;
279
290
  }
280
291
  }
@@ -384,44 +395,7 @@ export async function resolveSkill(input: ResolveSkillInput): Promise<ResolveRes
384
395
  return result;
385
396
  }
386
397
 
387
- const clients = getClients();
388
-
389
- const lexical = ftsSearch(db, input.query, config.recall.k_lexical);
390
-
391
- let retrieval: RetrievalCapability = "lexical";
392
- let rows = lexical;
393
- if (!input.forceLexical) {
394
- try {
395
- const queryVec = (await clients.embed([input.query]))[0]!;
396
- const nearest = vectorTopK(db, queryVec, config.recall.k_vector);
397
- rows = reciprocalRankFusion(lexical, nearest);
398
- retrieval = "hybrid";
399
- } catch {
400
- retrieval = "lexical";
401
- }
402
- }
403
-
404
- let scores: number[] | null = null;
405
- if (clients.rerank && retrieval === "hybrid" && rows.length > 0) {
406
- try {
407
- scores = await clients.rerank(
408
- input.query,
409
- rows.map((r) => ({ skill_id: r.skill_id, text: rerankText(r) })),
410
- );
411
- retrieval = "reranked";
412
- } catch {
413
- scores = null;
414
- }
415
- }
416
-
417
- const rankedCandidates: RankedCandidate[] = rows
418
- .map((r, i) => ({
419
- skill_id: r.skill_id,
420
- title: r.title,
421
- description: r.description,
422
- score: scores?.[i] ?? null,
423
- }))
424
- .sort((a, b) => scores === null ? 0 : (b.score ?? -Infinity) - (a.score ?? -Infinity));
398
+ const { retrieval, candidates: rankedCandidates } = await retrieveAndRerank(input);
425
399
 
426
400
  const decisionThresholds = retrieval === "reranked" && config.inference.mode === "remote"
427
401
  ? { candidate_limit: config.thresholds.candidate_limit, ...config.inference.thresholds }
@@ -473,6 +447,63 @@ export async function resolveSkill(input: ResolveSkillInput): Promise<ResolveRes
473
447
  return result;
474
448
  }
475
449
 
450
+ export interface RetrievalResult {
451
+ retrieval: Exclude<RetrievalCapability, "exact">;
452
+ candidates: RankedCandidate[];
453
+ }
454
+
455
+ /**
456
+ * Retrieve the full fused candidate set and rerank it once without applying
457
+ * decision thresholds. Calibration uses this to avoid observing the policy it
458
+ * is trying to replace.
459
+ */
460
+ export async function retrieveAndRerank(
461
+ input: ResolveSkillInput,
462
+ ): Promise<RetrievalResult> {
463
+ const { config, db } = await getEnv();
464
+ await syncVaultIfNeeded();
465
+ const clients = getClients();
466
+ const lexical = ftsSearch(db, input.query, config.recall.k_lexical);
467
+
468
+ let retrieval: RetrievalResult["retrieval"] = "lexical";
469
+ let rows = lexical;
470
+ if (!input.forceLexical) {
471
+ try {
472
+ const queryVec = (await clients.embed([input.query]))[0];
473
+ if (!queryVec) throw new Error("Embedding client returned no query vector.");
474
+ const nearest = vectorTopK(db, queryVec, config.recall.k_vector);
475
+ rows = reciprocalRankFusion(lexical, nearest);
476
+ retrieval = "hybrid";
477
+ } catch {
478
+ retrieval = "lexical";
479
+ }
480
+ }
481
+
482
+ let scores: number[] | null = null;
483
+ if (clients.rerank && retrieval === "hybrid" && rows.length > 0) {
484
+ try {
485
+ scores = await clients.rerank(
486
+ input.query,
487
+ rows.map((r) => ({ skill_id: r.skill_id, text: rerankText(r) })),
488
+ );
489
+ retrieval = "reranked";
490
+ } catch {
491
+ scores = null;
492
+ }
493
+ }
494
+
495
+ const candidates = rows
496
+ .map((r, i) => ({
497
+ skill_id: r.skill_id,
498
+ title: r.title,
499
+ description: r.description,
500
+ score: scores?.[i] ?? null,
501
+ }))
502
+ .sort((a, b) => scores === null ? 0 : (b.score ?? -Infinity) - (a.score ?? -Infinity));
503
+
504
+ return { retrieval, candidates };
505
+ }
506
+
476
507
  export async function fetchSkill(input: FetchSkillInput): Promise<FetchSkillResult> {
477
508
  const { config, db } = await getEnv();
478
509
  await syncVaultIfNeeded();