catherd-cli 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -14
- package/catalog/ATTRIBUTION.md +36 -0
- package/catalog/models.json +10 -0
- package/catalog/scores.json +3006 -194
- package/catalog/sources.json +86 -0
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/bin/catherd-mcp +1 -1
- package/plugin/skills/catherd/SKILL.md +2 -2
- package/src/domain/bars.ts +119 -0
- package/src/domain/calibration.ts +125 -0
- package/src/domain/catalog.ts +210 -23
- package/src/domain/failover.ts +14 -1
- package/src/domain/profile-rules.ts +108 -18
- package/src/domain/sources.ts +144 -0
- package/src/entry/catalog-command.ts +163 -8
- package/src/entry/init-command.ts +72 -2
- package/src/entry/mcp/lane-tools.ts +1 -1
- package/src/entry/mcp/server.ts +13 -3
- package/src/entry/mcp/setup-tools.ts +24 -2
- package/src/entry/profile-command.ts +10 -4
- package/src/entry/tui/commands.ts +20 -2
- package/src/entry/tui/effects.ts +50 -1
- package/src/entry/tui/fixtures.ts +31 -3
- package/src/entry/tui/profile-edits.ts +17 -9
- package/src/entry/tui/profile-tree.ts +2 -2
- package/src/entry/tui/state.ts +4 -0
- package/src/entry/tui/views/profile-actions.ts +9 -1
- package/src/entry/tui/views/profiles.tsx +81 -14
- package/src/entry/tui/views/save-dialog.tsx +12 -4
- package/src/infra/env.ts +2 -0
- package/src/infra/jev-client.ts +88 -40
- package/src/infra/paths.ts +2 -0
- package/src/infra/sources/arena.ts +87 -0
- package/src/infra/sources/artificial-analysis.ts +112 -0
- package/src/infra/sources/cache.ts +124 -0
- package/src/infra/sources/csv.ts +38 -0
- package/src/infra/sources/epoch.ts +82 -0
- package/src/infra/sources/http.ts +75 -0
- package/src/infra/sources/litellm.ts +58 -0
- package/src/infra/sources/models-dev.ts +59 -0
- package/src/infra/sources/openrouter-endpoints.ts +52 -0
- package/src/infra/sources/openrouter-models.ts +60 -0
- package/src/infra/sources/rows.ts +38 -0
- package/src/infra/sources/vectara.ts +69 -0
- package/src/infra/sources/zip.ts +48 -0
- package/src/services/catalog-refresh.ts +200 -0
- package/src/services/catalog-service.ts +83 -23
- package/src/services/credentials.ts +69 -0
- package/src/services/doctor-sources.ts +76 -0
- package/src/services/doctor.ts +10 -0
- package/src/services/jev-service.ts +14 -41
- package/src/services/ports.ts +6 -0
- package/src/services/profile-service.ts +8 -6
- package/src/services/provenance.ts +115 -0
- package/src/services/routing-service.ts +7 -2
- package/src/services/run-evidence.ts +86 -0
- package/src/services/source-derive.ts +349 -0
- package/src/services/source-sync.ts +305 -0
- package/src/services/standins.ts +204 -0
- package/src/services/treat-likes.ts +84 -0
package/src/domain/catalog.ts
CHANGED
|
@@ -3,10 +3,31 @@ import { type Rung, parseRung } from "./ids.ts";
|
|
|
3
3
|
import { DIFFICULTIES, type Difficulty, KINDS, type Kind } from "./lane.ts";
|
|
4
4
|
import type { Role } from "./roles.ts";
|
|
5
5
|
|
|
6
|
-
/**
|
|
7
|
-
|
|
6
|
+
/**
|
|
7
|
+
* Spec §5.2 and 1.2 §5.1: the scored dimensions, each in its anchor's unit. `agentic`, `steer` and `frontend`
|
|
8
|
+
* (1.2) carry no shipped bar yet.
|
|
9
|
+
*/
|
|
10
|
+
export const DIMS = ["repo_code", "terminal", "honesty", "agentic", "steer", "frontend"] as const;
|
|
8
11
|
export type Dim = (typeof DIMS)[number];
|
|
9
12
|
|
|
13
|
+
/**
|
|
14
|
+
* Spec 1.2 §4.3: how a value was obtained, best first. The 1.0 levels (`verified`, `secondary`,
|
|
15
|
+
* `inferred`) keep their names, so an older override file stays valid.
|
|
16
|
+
*/
|
|
17
|
+
export const CONFIDENCE = [
|
|
18
|
+
"verified",
|
|
19
|
+
"measured",
|
|
20
|
+
"calibrated",
|
|
21
|
+
"adjacent",
|
|
22
|
+
"secondary",
|
|
23
|
+
"inferred",
|
|
24
|
+
] as const;
|
|
25
|
+
export type Confidence = (typeof CONFIDENCE)[number];
|
|
26
|
+
export const RANK = Object.fromEntries(CONFIDENCE.map((c, i) => [c, i])) as Record<Confidence, number>;
|
|
27
|
+
/** Spec 1.2 §4.3: a value older than this many days counts one level lower. */
|
|
28
|
+
export const STALE_DAYS = 90;
|
|
29
|
+
const DAY_MS = 86_400_000;
|
|
30
|
+
|
|
10
31
|
/**
|
|
11
32
|
* Spec §7.1 `billing` keys: a rung's backend, except that opencode's Go models (`opencode-go/…`) are billed
|
|
12
33
|
* apart from its Zen models (`opencode/…`).
|
|
@@ -52,6 +73,10 @@ const FamilySchema = z.looseObject({
|
|
|
52
73
|
meteredOnPlan: z.boolean().default(false),
|
|
53
74
|
on: z.partialRecord(z.enum(MODEL_KEYS), BackendModelSchema),
|
|
54
75
|
notes: z.record(z.string(), z.string()).default({}),
|
|
76
|
+
/** the day the vendor released it (models.dev), a stand-in feature (spec 1.2 §6.3) */
|
|
77
|
+
releaseDate: z.iso.date().optional(),
|
|
78
|
+
/** a sync's speed facts (spec 1.2 §4.1), `<source>.<field>` → value; they never carry a bar */
|
|
79
|
+
speed: z.record(z.string(), z.number()).optional(),
|
|
55
80
|
});
|
|
56
81
|
export type Family = z.infer<typeof FamilySchema>;
|
|
57
82
|
|
|
@@ -72,7 +97,18 @@ export type ModelsFile = z.infer<typeof ModelsFileSchema>;
|
|
|
72
97
|
/** A canonical rung: `<canonical model id>#<effort>`, the key scores and treat-likes use. */
|
|
73
98
|
export const CanonicalRung = z.string().regex(/^[^:#\s]+#[^#\s]+$/, "a canonical rung is model#effort");
|
|
74
99
|
|
|
75
|
-
|
|
100
|
+
/** Spec 1.2 §4.2: `anchor = a·x + b`, fitted on `n` rungs both sources cover, with its R². */
|
|
101
|
+
export const FitSchema = z.object({
|
|
102
|
+
source: z.string(),
|
|
103
|
+
field: z.string(),
|
|
104
|
+
a: z.number(),
|
|
105
|
+
b: z.number(),
|
|
106
|
+
r2: z.number(),
|
|
107
|
+
n: z.number().int(),
|
|
108
|
+
});
|
|
109
|
+
export type Fit = z.infer<typeof FitSchema>;
|
|
110
|
+
|
|
111
|
+
export const ScoreSchema = z.object({
|
|
76
112
|
rung: CanonicalRung,
|
|
77
113
|
dim: z.enum(DIMS),
|
|
78
114
|
value: z.number(),
|
|
@@ -80,8 +116,14 @@ const ScoreSchema = z.object({
|
|
|
80
116
|
version: z.string(),
|
|
81
117
|
url: z.url(),
|
|
82
118
|
date: z.iso.date(),
|
|
83
|
-
confidence: z.enum(
|
|
119
|
+
confidence: z.enum(CONFIDENCE),
|
|
84
120
|
note: z.string().optional(),
|
|
121
|
+
/** spec 1.2 §4.3: the source a synced value came from (absent: the shipped file or the user's override) */
|
|
122
|
+
source: z.string().optional(),
|
|
123
|
+
/** a calibrated value: the fit that mapped it onto the anchor */
|
|
124
|
+
fit: FitSchema.optional(),
|
|
125
|
+
/** the source named no effort, so the family's default effort was assumed (spec 1.2 §3.4) */
|
|
126
|
+
effortAssumed: z.boolean().optional(),
|
|
85
127
|
});
|
|
86
128
|
export type Score = z.infer<typeof ScoreSchema>;
|
|
87
129
|
|
|
@@ -89,7 +131,13 @@ const BarSchema = z.partialRecord(z.enum(DIMS), z.number());
|
|
|
89
131
|
type Bars = Record<Kind, Record<Difficulty, Partial<Record<Dim, number>>>>;
|
|
90
132
|
const BarsSchema = z.record(z.enum(KINDS), z.record(z.enum(DIFFICULTIES), BarSchema));
|
|
91
133
|
|
|
92
|
-
/**
|
|
134
|
+
/** Spec 1.2 §5.2: per dimension and difficulty, where the default threshold came from. */
|
|
135
|
+
const BarsWhySchema = z.partialRecord(z.enum(DIMS), z.partialRecord(z.enum(DIFFICULTIES), z.string()));
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* `catalog/scores.json`: the hand-typed and keyless values (spec 1.2 §7), the shipped treat-likes, and the
|
|
139
|
+
* default bars with the `barsWhy` line of each threshold (spec 1.2 §5.2).
|
|
140
|
+
*/
|
|
93
141
|
export const ScoresFileSchema = z.looseObject({
|
|
94
142
|
schema: z.literal(1),
|
|
95
143
|
version: z.string(),
|
|
@@ -97,18 +145,35 @@ export const ScoresFileSchema = z.looseObject({
|
|
|
97
145
|
scores: z.array(ScoreSchema),
|
|
98
146
|
treatLike: z.record(CanonicalRung, z.object({ like: CanonicalRung, note: z.string() })),
|
|
99
147
|
bars: BarsSchema,
|
|
148
|
+
barsWhy: BarsWhySchema.default({}),
|
|
100
149
|
});
|
|
101
150
|
export type ScoresFile = z.infer<typeof ScoresFileSchema>;
|
|
102
151
|
|
|
103
|
-
/**
|
|
152
|
+
/** An override's bar: per dimension a threshold, or `null` to remove the default's (spec 1.2 §5.2). */
|
|
153
|
+
const OverrideBarSchema = z.partialRecord(z.enum(DIMS), z.number().nullable());
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* `<config>/catalog.override.json` (spec §3.5): the user's treat-likes, scores and bars. Its bars override
|
|
157
|
+
* the default bars per dimension (spec 1.2 §5.2): a number sets that threshold, `null` removes it, and a
|
|
158
|
+
* dimension it does not name keeps the default's.
|
|
159
|
+
*/
|
|
104
160
|
export const OverrideSchema = z.looseObject({
|
|
105
161
|
schema: z.literal(1).default(1),
|
|
106
162
|
treatLike: z.record(CanonicalRung, CanonicalRung).default({}),
|
|
107
163
|
scores: z.array(ScoreSchema).default([]),
|
|
108
|
-
bars: z.partialRecord(z.enum(KINDS), z.partialRecord(z.enum(DIFFICULTIES),
|
|
164
|
+
bars: z.partialRecord(z.enum(KINDS), z.partialRecord(z.enum(DIFFICULTIES), OverrideBarSchema)).default({}),
|
|
109
165
|
});
|
|
110
166
|
export type Override = z.infer<typeof OverrideSchema>;
|
|
111
167
|
|
|
168
|
+
/** Spec 1.2 §6.1: the rung whose value a rung without one uses on a dimension, as `inferred`. */
|
|
169
|
+
export interface InferredStandIn {
|
|
170
|
+
like: string;
|
|
171
|
+
/** spec 1.2 §6.3's distance between the two */
|
|
172
|
+
distance: number;
|
|
173
|
+
/** the features that distance rests on */
|
|
174
|
+
features: string[];
|
|
175
|
+
}
|
|
176
|
+
|
|
112
177
|
interface Listed {
|
|
113
178
|
id: string;
|
|
114
179
|
efforts: string[];
|
|
@@ -124,29 +189,119 @@ export interface Catalog {
|
|
|
124
189
|
scores: Record<string, Partial<Record<Dim, Score>>>;
|
|
125
190
|
treatLike: Record<string, { like: string; source: "shipped" | "user" }>;
|
|
126
191
|
bars: Bars;
|
|
192
|
+
/** where each default threshold came from (spec 1.2 §5.2), by dimension and difficulty */
|
|
193
|
+
barsWhy: ScoresFile["barsWhy"];
|
|
127
194
|
/** per backend id, the last `listModels()`; absent when never listed */
|
|
128
195
|
listed: Record<string, { fetchedAt: string; models: Listed[] }>;
|
|
129
196
|
/** `<canonical rung>|<kind or *>` → median seconds, only with ≥ 5 samples (spec §5.2) */
|
|
130
197
|
secs: Record<string, number>;
|
|
198
|
+
/** canonical rung → the Artificial Analysis stand-in features a sync read for it (spec 1.2 §6.3) */
|
|
199
|
+
features: Record<string, Record<string, number>>;
|
|
200
|
+
/**
|
|
201
|
+
* canonical rung → per dimension it has no value for (of its own or through a treat-like), the stand-in
|
|
202
|
+
* whose value it uses as `inferred` (spec 1.2 §6.1); filled by the stand-in ranking (services/standins.ts)
|
|
203
|
+
*/
|
|
204
|
+
inferred: Record<string, Partial<Record<Dim, InferredStandIn>>>;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Spec 1.2 §3.5: what a sync learned about a family. `price`, `capabilities` and the opencode backends'
|
|
209
|
+
* `efforts` and `context` come from models.dev; `speed` holds facts that never carry a bar (spec 1.2 §4.1),
|
|
210
|
+
* as `<source>.<field>` → value.
|
|
211
|
+
*/
|
|
212
|
+
export const FamilyFactsSchema = z.object({
|
|
213
|
+
price: z.object({ input: z.number(), cached: z.number(), output: z.number() }).optional(),
|
|
214
|
+
capabilities: z.object({ toolUse: z.boolean(), imageIn: z.boolean(), reasoning: z.boolean() }).optional(),
|
|
215
|
+
on: z
|
|
216
|
+
.partialRecord(
|
|
217
|
+
z.enum(MODEL_KEYS),
|
|
218
|
+
z.object({ efforts: z.array(z.string()).optional(), context: z.number().int().positive().optional() }),
|
|
219
|
+
)
|
|
220
|
+
.default({}),
|
|
221
|
+
releaseDate: z.string().optional(),
|
|
222
|
+
speed: z.record(z.string(), z.number()).default({}),
|
|
223
|
+
});
|
|
224
|
+
export type FamilyFacts = z.infer<typeof FamilyFactsSchema>;
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* Spec 1.2 §3.5: the families with a sync's facts laid over them. The shipped file stays the floor: a fact
|
|
228
|
+
* the sync lacks keeps the shipped one, and a backend keeps every effort it ships with.
|
|
229
|
+
*/
|
|
230
|
+
export function applyFacts(families: Family[], facts: Record<string, FamilyFacts>): Family[] {
|
|
231
|
+
return families.map((f) => {
|
|
232
|
+
const x = facts[f.id];
|
|
233
|
+
if (!x) return f;
|
|
234
|
+
const on = { ...f.on };
|
|
235
|
+
for (const key of MODEL_KEYS) {
|
|
236
|
+
const cur = on[key];
|
|
237
|
+
const got = x.on[key];
|
|
238
|
+
if (!cur || !got) continue;
|
|
239
|
+
on[key] = {
|
|
240
|
+
...cur,
|
|
241
|
+
efforts: [...cur.efforts, ...(got.efforts ?? []).filter((e) => !cur.efforts.includes(e))],
|
|
242
|
+
context: got.context ?? cur.context,
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
// a sync may add a capability, never remove one the shipped file gives (the floor)
|
|
246
|
+
const got = x.capabilities;
|
|
247
|
+
const capabilities = got
|
|
248
|
+
? {
|
|
249
|
+
toolUse: f.capabilities.toolUse || got.toolUse,
|
|
250
|
+
imageIn: f.capabilities.imageIn || got.imageIn,
|
|
251
|
+
reasoning: f.capabilities.reasoning || got.reasoning,
|
|
252
|
+
}
|
|
253
|
+
: f.capabilities;
|
|
254
|
+
return {
|
|
255
|
+
...f,
|
|
256
|
+
price: x.price ?? f.price,
|
|
257
|
+
capabilities,
|
|
258
|
+
on,
|
|
259
|
+
...(x.releaseDate || f.releaseDate ? { releaseDate: x.releaseDate ?? f.releaseDate } : {}),
|
|
260
|
+
...(Object.keys(x.speed).length ? { speed: x.speed } : {}),
|
|
261
|
+
};
|
|
262
|
+
});
|
|
131
263
|
}
|
|
132
264
|
|
|
133
|
-
|
|
265
|
+
/** Spec 1.2 §4.3: the level a value counts at: its own, one lower once it is older than STALE_DAYS. */
|
|
266
|
+
export function effectiveRank(s: Score, now: number): number {
|
|
267
|
+
const stale = now - Date.parse(s.date) > STALE_DAYS * DAY_MS;
|
|
268
|
+
return Math.min(RANK[s.confidence] + (stale ? 1 : 0), CONFIDENCE.length - 1);
|
|
269
|
+
}
|
|
134
270
|
|
|
271
|
+
/** Spec 1.2 §4.3: `a` beats `b` at a better level, or at the same level with a newer date. */
|
|
272
|
+
export function outranks(a: Score, b: Score, now: number): boolean {
|
|
273
|
+
const ra = effectiveRank(a, now);
|
|
274
|
+
const rb = effectiveRank(b, now);
|
|
275
|
+
return ra < rb || (ra === rb && a.date > b.date);
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* The catalog routing reads: the shipped files, the synced values (spec 1.2 §3), each backend's listing and
|
|
280
|
+
* the user's override, which always wins. Between shipped and synced values the better level wins, then the
|
|
281
|
+
* newer date (spec 1.2 §4.3); `now` dates the 90-day drop.
|
|
282
|
+
*/
|
|
135
283
|
export function buildCatalog(o: {
|
|
136
284
|
models: ModelsFile;
|
|
137
285
|
scores: ScoresFile;
|
|
286
|
+
synced?: Score[];
|
|
287
|
+
facts?: Record<string, FamilyFacts>;
|
|
138
288
|
override?: Override;
|
|
139
289
|
listed?: Catalog["listed"];
|
|
140
290
|
secs?: Catalog["secs"];
|
|
291
|
+
features?: Catalog["features"];
|
|
292
|
+
now?: number;
|
|
141
293
|
}): Catalog {
|
|
294
|
+
const now = o.now ?? Date.now();
|
|
142
295
|
const scores: Catalog["scores"] = {};
|
|
143
296
|
const put = (s: Score, force: boolean) => {
|
|
144
297
|
const cur = (scores[s.rung] ??= {});
|
|
145
298
|
const had = cur[s.dim];
|
|
146
|
-
if (force || !had ||
|
|
299
|
+
if (force || !had || outranks(s, had, now)) cur[s.dim] = s;
|
|
147
300
|
};
|
|
148
301
|
for (const s of o.scores.scores) put(s, false);
|
|
149
|
-
for (const s of o.
|
|
302
|
+
for (const s of o.synced ?? []) put(s, false);
|
|
303
|
+
// the user's values say so whatever source they were copied with, for route's provenance (spec 1.2 §5.3)
|
|
304
|
+
for (const s of o.override?.scores ?? []) put({ ...s, source: "override" }, true);
|
|
150
305
|
const treatLike: Catalog["treatLike"] = {};
|
|
151
306
|
for (const [rung, t] of Object.entries(o.scores.treatLike))
|
|
152
307
|
treatLike[rung] = { like: t.like, source: "shipped" };
|
|
@@ -154,18 +309,21 @@ export function buildCatalog(o: {
|
|
|
154
309
|
treatLike[rung] = { like, source: "user" };
|
|
155
310
|
const bars = structuredClone(o.scores.bars) as Bars;
|
|
156
311
|
for (const kind of KINDS)
|
|
157
|
-
for (const d of DIFFICULTIES)
|
|
158
|
-
const
|
|
159
|
-
|
|
160
|
-
|
|
312
|
+
for (const d of DIFFICULTIES)
|
|
313
|
+
for (const [dim, min] of Object.entries(o.override?.bars[kind]?.[d] ?? {}) as [Dim, number | null][])
|
|
314
|
+
if (min === null) delete bars[kind][d][dim];
|
|
315
|
+
else bars[kind][d][dim] = min;
|
|
161
316
|
return {
|
|
162
|
-
families: o.models.families,
|
|
317
|
+
families: o.facts ? applyFacts(o.models.families, o.facts) : o.models.families,
|
|
163
318
|
backends: o.models.backends,
|
|
164
319
|
scores,
|
|
165
320
|
treatLike,
|
|
166
321
|
bars,
|
|
322
|
+
barsWhy: o.scores.barsWhy,
|
|
167
323
|
listed: o.listed ?? {},
|
|
168
324
|
secs: o.secs ?? {},
|
|
325
|
+
features: o.features ?? {},
|
|
326
|
+
inferred: {},
|
|
169
327
|
};
|
|
170
328
|
}
|
|
171
329
|
|
|
@@ -211,18 +369,47 @@ export function rungInfo(c: Catalog, rung: string): RungInfo {
|
|
|
211
369
|
};
|
|
212
370
|
}
|
|
213
371
|
|
|
214
|
-
/**
|
|
372
|
+
/**
|
|
373
|
+
* The rung's scores: per dimension its own value, else that of the rung it is treated like (a sync may score
|
|
374
|
+
* a rung on some dimensions only), else its inferred stand-in's (spec 1.2 §6.1). `via` names the treat-like
|
|
375
|
+
* when it lends any value and `borrowed` the dimensions it lends; `inferred` the dimensions an inferred
|
|
376
|
+
* stand-in fills and `standIns` whose values they are. null when it has no value at all.
|
|
377
|
+
*/
|
|
215
378
|
export function scoresOf(
|
|
216
379
|
c: Catalog,
|
|
217
380
|
canonical: string,
|
|
218
|
-
): {
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
381
|
+
): {
|
|
382
|
+
values: Partial<Record<Dim, number>>;
|
|
383
|
+
records: Partial<Record<Dim, Score>>;
|
|
384
|
+
via: string | null;
|
|
385
|
+
borrowed: Dim[];
|
|
386
|
+
inferred: Dim[];
|
|
387
|
+
standIns: Partial<Record<Dim, string>>;
|
|
388
|
+
} | null {
|
|
389
|
+
const own = c.scores[canonical] ?? {};
|
|
390
|
+
const like = c.treatLike[canonical]?.like ?? null;
|
|
391
|
+
const lent = like ? (c.scores[like] ?? {}) : {};
|
|
392
|
+
const guessed = c.inferred[canonical] ?? {};
|
|
223
393
|
const values: Partial<Record<Dim, number>> = {};
|
|
224
|
-
|
|
225
|
-
|
|
394
|
+
const records: Partial<Record<Dim, Score>> = {};
|
|
395
|
+
const borrowed: Dim[] = [];
|
|
396
|
+
const inferred: Dim[] = [];
|
|
397
|
+
const standIns: Partial<Record<Dim, string>> = {};
|
|
398
|
+
for (const d of DIMS) {
|
|
399
|
+
const stand = guessed[d];
|
|
400
|
+
const r = own[d] ?? lent[d] ?? (stand ? c.scores[stand.like]?.[d] : undefined);
|
|
401
|
+
if (!r) continue;
|
|
402
|
+
records[d] = r;
|
|
403
|
+
values[d] = r.value;
|
|
404
|
+
if (own[d]) continue;
|
|
405
|
+
if (lent[d]) borrowed.push(d);
|
|
406
|
+
else if (stand) {
|
|
407
|
+
inferred.push(d);
|
|
408
|
+
standIns[d] = stand.like;
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
if (Object.keys(records).length === 0) return null;
|
|
412
|
+
return { values, records, via: borrowed.length ? like : null, borrowed, inferred, standIns };
|
|
226
413
|
}
|
|
227
414
|
|
|
228
415
|
/** Spec §4 roles: what a rung must offer to be placed on a role. */
|
package/src/domain/failover.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { billingKeyOf, type Catalog, DIMS, type Dim, rungInfo, scoresOf } from "
|
|
|
2
2
|
import { type BillingMode, compareCost, type Cost, costOf, DEFAULT_BILLING } from "./cost.ts";
|
|
3
3
|
import { type Rung, tryParseRung } from "./ids.ts";
|
|
4
4
|
import { DIFFICULTIES, KINDS } from "./lane.ts";
|
|
5
|
+
import { EFFORT_ORDER } from "./sources.ts";
|
|
5
6
|
|
|
6
7
|
/**
|
|
7
8
|
* Spec §7.1 and §4.5: a stand-in on the same quota would be out of quota too. The billing key names the
|
|
@@ -72,7 +73,9 @@ const costFor = (c: Catalog, billing: Partial<Record<string, BillingMode>>, rung
|
|
|
72
73
|
* Spec 1.1 §11: the rungs of `pool` that can stand in for `rung`, best first. A stand-in is on another
|
|
73
74
|
* quota, scored, paid from a plan or subscription under `billing` (a metered one spends money nobody chose
|
|
74
75
|
* to), startable by `dispatch` (not a native `claude:` subagent), and no downgrade on the rung's bar dims.
|
|
75
|
-
* Claude-billed stand-ins rank last, then the
|
|
76
|
+
* Claude-billed stand-ins rank last, then the effort nearest the rung's own (a model's efforts carry one
|
|
77
|
+
* another's values as `adjacent`, spec 1.2 §4.3, so the cheapest would otherwise be its lowest effort; plan 14
|
|
78
|
+
* Ruling 4), then the cheapest.
|
|
76
79
|
*/
|
|
77
80
|
export function rankStandIns(
|
|
78
81
|
c: Catalog,
|
|
@@ -88,14 +91,24 @@ export function rankStandIns(
|
|
|
88
91
|
if (costFor(c, billing, x).tier === 1) return false;
|
|
89
92
|
return downgradeDims(c, rung, x).length === 0;
|
|
90
93
|
});
|
|
94
|
+
const gap = (x: string) => effortGap(from.effort, tryParseRung(x)?.effort ?? "");
|
|
91
95
|
return fits.sort(
|
|
92
96
|
(a, b) =>
|
|
93
97
|
Number(claudeBilled(a)) - Number(claudeBilled(b)) ||
|
|
98
|
+
gap(a) - gap(b) ||
|
|
94
99
|
compareCost(costFor(c, billing, a), costFor(c, billing, b)) ||
|
|
95
100
|
a.localeCompare(b),
|
|
96
101
|
);
|
|
97
102
|
}
|
|
98
103
|
|
|
104
|
+
/** How many effort steps apart two efforts are; an effort that is no effort word (`default`) is far from all. */
|
|
105
|
+
function effortGap(a: string, b: string): number {
|
|
106
|
+
const i = (EFFORT_ORDER as readonly string[]).indexOf(a);
|
|
107
|
+
const j = (EFFORT_ORDER as readonly string[]).indexOf(b);
|
|
108
|
+
if (a === b) return 0;
|
|
109
|
+
return i < 0 || j < 0 ? EFFORT_ORDER.length : Math.abs(i - j);
|
|
110
|
+
}
|
|
111
|
+
|
|
99
112
|
/** How a family's `on` key or a model id's prefix maps to the backend that runs it. */
|
|
100
113
|
const BACKEND_OF_KEY: Record<string, string> = { "opencode-go": "opencode" };
|
|
101
114
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
type Catalog,
|
|
3
|
+
type Dim,
|
|
3
4
|
capableFor,
|
|
4
5
|
effortOffered,
|
|
5
6
|
ROLE_NEEDS,
|
|
@@ -37,6 +38,71 @@ export interface Validation {
|
|
|
37
38
|
|
|
38
39
|
const TREAT_LIKE_FIX = (rung: string) =>
|
|
39
40
|
`map it with: catherd catalog treat-like ${rung} <a scored rung>; catalog_query lists them`;
|
|
41
|
+
const SUGGEST_FIX = (rung: string) =>
|
|
42
|
+
`confirm or replace it: catherd catalog treat-like --suggest ${rung}, then catherd catalog treat-like ${rung} <a rung>`;
|
|
43
|
+
|
|
44
|
+
/** Spec 1.2 §6.2: two findings are the same one when they name the same path with the same message. */
|
|
45
|
+
const sameIssue = (a: Issue, b: Issue): boolean => a.path === b.path && a.message === b.message;
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Spec 1.2 §6.2: a save that leaves errors goes through when it removes at least one of the errors the
|
|
49
|
+
* profile had and adds none. `before` is the stored profile's validation, null when it does not exist yet.
|
|
50
|
+
*/
|
|
51
|
+
export function repairs(before: Validation | null, after: Validation): boolean {
|
|
52
|
+
if (after.errors.length === 0) return true;
|
|
53
|
+
if (!before) return false;
|
|
54
|
+
const added = after.errors.some((e) => !before.errors.some((b) => sameIssue(b, e)));
|
|
55
|
+
const removed = before.errors.some((b) => !after.errors.some((e) => sameIssue(b, e)));
|
|
56
|
+
return removed && !added;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Spec 1.2 §6.1 "stand-in to confirm": a rung whose values on some bar dimensions are an inferred stand-in's. */
|
|
60
|
+
export interface StandInToConfirm {
|
|
61
|
+
canonical: string;
|
|
62
|
+
/** the profile's rungs that run it, and where the first one is */
|
|
63
|
+
rungs: string[];
|
|
64
|
+
path: string;
|
|
65
|
+
dims: { dim: Dim; like: string }[];
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Spec 1.2 §6.1: the profile's rungs that lean on an inferred stand-in, one entry per canonical rung. Only
|
|
70
|
+
* where bars choose: rungs of a role with more than one usable rung, and failover stand-ins; a role with one
|
|
71
|
+
* rung runs it whatever the bars say (plan 14 Ruling 13).
|
|
72
|
+
*/
|
|
73
|
+
export function standInsToConfirm(p: Profile, c: Catalog, backends: readonly string[]): StandInToConfirm[] {
|
|
74
|
+
const seen = new Map<string, StandInToConfirm>();
|
|
75
|
+
const add = (rung: string, path: string) => {
|
|
76
|
+
const r = tryParseRung(rung);
|
|
77
|
+
if (!r || !backends.includes(r.backend)) return;
|
|
78
|
+
const info = rungInfo(c, rung);
|
|
79
|
+
const s = scoresOf(c, info.canonical);
|
|
80
|
+
if (!s || s.inferred.length === 0) return;
|
|
81
|
+
const had = seen.get(info.canonical);
|
|
82
|
+
if (had) {
|
|
83
|
+
if (!had.rungs.includes(rung)) had.rungs.push(rung);
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
const dims = s.inferred.map((dim) => ({ dim, like: s.standIns[dim] as string }));
|
|
87
|
+
seen.set(info.canonical, { canonical: info.canonical, rungs: [rung], path, dims });
|
|
88
|
+
};
|
|
89
|
+
for (const role of ROLES) {
|
|
90
|
+
const rc = p.roles[role];
|
|
91
|
+
if (!rc.enabled || candidates(c, routingProfileOf(p, role), role).length < 2) continue;
|
|
92
|
+
for (const rung of rc.rungs) add(rung, `roles.${role}.rungs`);
|
|
93
|
+
}
|
|
94
|
+
for (const [from, to] of Object.entries(p.failover)) add(to, `failover.${from}`);
|
|
95
|
+
return [...seen.values()];
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** One "stand-in to confirm" warning's words. */
|
|
99
|
+
export function standInMessage(x: StandInToConfirm): string {
|
|
100
|
+
const words = (xs: string[]) =>
|
|
101
|
+
xs.length < 3 ? xs.join(" or ") : `${xs.slice(0, -1).join(", ")} or ${xs.at(-1)}`;
|
|
102
|
+
const lent = x.dims.map((d) => `${d.like}'s ${d.dim}`).join(", ");
|
|
103
|
+
const also = x.rungs.length > 1 ? ` (${x.rungs.join(", ")})` : "";
|
|
104
|
+
return `stand-in to confirm: ${x.canonical}${also} has no ${words(x.dims.map((d) => d.dim))} value of its own; routing uses ${lent} (inferred)`;
|
|
105
|
+
}
|
|
40
106
|
|
|
41
107
|
export const routingProfileOf = (p: Profile, role: Role): RoutingProfile => ({
|
|
42
108
|
objective: p.objective,
|
|
@@ -44,22 +110,35 @@ export const routingProfileOf = (p: Profile, role: Role): RoutingProfile => ({
|
|
|
44
110
|
role: p.roles[role],
|
|
45
111
|
});
|
|
46
112
|
|
|
47
|
-
/**
|
|
48
|
-
|
|
113
|
+
/**
|
|
114
|
+
* Whether a rung's scores are catherd's guess: borrowed through a treat-like, filled by an inferred stand-in
|
|
115
|
+
* (spec 1.2 §6.1), or only `inferred` ones. `note` says what a treat-like lends: "scores borrowed from X"
|
|
116
|
+
* when the rung has no value of its own, else the dimensions it borrows (a rung a sync scored on some).
|
|
117
|
+
*/
|
|
118
|
+
export function inferredScores(
|
|
119
|
+
c: Catalog,
|
|
120
|
+
info: RungInfo,
|
|
121
|
+
): { inferred: boolean; via: string | null; note: string | null } {
|
|
49
122
|
const s = scoresOf(c, info.canonical);
|
|
50
|
-
if (!s) return { inferred: false, via: null };
|
|
123
|
+
if (!s) return { inferred: false, via: null, note: null };
|
|
51
124
|
const records = Object.values(s.records);
|
|
52
|
-
|
|
125
|
+
const guessed =
|
|
126
|
+
s.via !== null || s.inferred.length > 0 || records.every((r) => r.confidence === "inferred");
|
|
127
|
+
const all = s.borrowed.length + s.inferred.length === records.length;
|
|
128
|
+
const note = s.via ? `${all ? "scores" : s.borrowed.join(", ")} borrowed from ${s.via}` : null;
|
|
129
|
+
return { inferred: guessed, via: s.via, note };
|
|
53
130
|
}
|
|
54
131
|
|
|
55
132
|
/**
|
|
56
|
-
* Spec §7.1. Errors: the worker disabled; an enabled role with no usable rung;
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
133
|
+
* Spec §7.1. Errors: the worker disabled; an enabled role with no usable rung; a failover stand-in on the
|
|
134
|
+
* same quota; a rung whose backend catherd cannot run (`backends` lists those it can, `claude` included).
|
|
135
|
+
* Everything else worth knowing is a warning: an access mode other than the role's default, an effort or
|
|
136
|
+
* model the last listing does not offer, a stand-in that never runs, (spec 1.1 §11) a stand-in that
|
|
137
|
+
* downgrades its rung, one that spends Claude quota while another plan could stand in, and a ladder that goes
|
|
138
|
+
* down, and (spec 1.2 §6.1) an unscored rung or stand-in, never an error: a rung that leans on an inferred
|
|
139
|
+
* stand-in is a "stand-in to confirm", and one with no value and no stand-in is skipped by routing. With the
|
|
140
|
+
* stored `doc` `p` came from, a value this catherd does not know is also a warning, one that says how the
|
|
141
|
+
* value is read.
|
|
63
142
|
*/
|
|
64
143
|
export function validateProfile(
|
|
65
144
|
p: Profile,
|
|
@@ -133,7 +212,11 @@ export function validateProfile(
|
|
|
133
212
|
message: `${r.backend}'s last listing does not offer ${r.model}; routing skips it`,
|
|
134
213
|
});
|
|
135
214
|
if (!scoresOf(c, info.canonical))
|
|
136
|
-
|
|
215
|
+
warnings.push({
|
|
216
|
+
path: `${at}.rungs`,
|
|
217
|
+
message: `${rung} is unscored and no rung is near enough to stand in for it: routing skips it`,
|
|
218
|
+
fix: TREAT_LIKE_FIX(rung),
|
|
219
|
+
});
|
|
137
220
|
}
|
|
138
221
|
rc.rungs.forEach((rung, i) => {
|
|
139
222
|
const prev = rc.rungs[i - 1];
|
|
@@ -170,14 +253,15 @@ export function validateProfile(
|
|
|
170
253
|
path: `${at}.rungs`,
|
|
171
254
|
message: `${x.rung} clears no routing bar, so a lane starts on it only as the role's default rung and never climbs onto it`,
|
|
172
255
|
});
|
|
256
|
+
// spec 1.2 §5.2: the logic and hard bars sit at the 60th and 75th percentiles, which a cost-minded ladder
|
|
257
|
+
// may never reach, and those lanes start at the default rung and climb; only a kind whose every bar the
|
|
258
|
+
// ladder misses routes blind (plan 14 Ruling 12)
|
|
173
259
|
if (role === "worker" && usable.length > 1) {
|
|
174
|
-
const
|
|
175
|
-
|
|
176
|
-
);
|
|
177
|
-
if (missed.length)
|
|
260
|
+
const blind = KINDS.filter((k) => !DIFFICULTIES.some((d) => usable.some((x) => clearsBar(c, x, k, d))));
|
|
261
|
+
if (blind.length)
|
|
178
262
|
warnings.push({
|
|
179
263
|
path: `${at}.rungs`,
|
|
180
|
-
message: `no worker rung clears
|
|
264
|
+
message: `no worker rung clears any bar for ${blind.join(", ")}; those lanes always start at the default rung`,
|
|
181
265
|
});
|
|
182
266
|
}
|
|
183
267
|
}
|
|
@@ -205,7 +289,11 @@ export function validateProfile(
|
|
|
205
289
|
continue;
|
|
206
290
|
}
|
|
207
291
|
if (!scoresOf(c, rungInfo(c, to).canonical))
|
|
208
|
-
|
|
292
|
+
warnings.push({
|
|
293
|
+
path: at,
|
|
294
|
+
message: `stand-in ${to} is unscored and no rung is near enough to stand in for it`,
|
|
295
|
+
fix: TREAT_LIKE_FIX(to),
|
|
296
|
+
});
|
|
209
297
|
if (quotaOf(a) === quotaOf(b))
|
|
210
298
|
errors.push({
|
|
211
299
|
path: at,
|
|
@@ -235,5 +323,7 @@ export function validateProfile(
|
|
|
235
323
|
if (!ladders.has(from))
|
|
236
324
|
warnings.push({ path: at, message: `${from} is on no enabled role's ladder, so this never runs` });
|
|
237
325
|
}
|
|
326
|
+
for (const x of standInsToConfirm(p, c, backends))
|
|
327
|
+
warnings.push({ path: x.path, message: standInMessage(x), fix: SUGGEST_FIX(x.rungs[0] as string) });
|
|
238
328
|
return { errors, warnings };
|
|
239
329
|
}
|