catherd-cli 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -14
- package/catalog/ATTRIBUTION.md +36 -0
- package/catalog/models.json +10 -0
- package/catalog/scores.json +3006 -194
- package/catalog/sources.json +86 -0
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/bin/catherd-mcp +1 -1
- package/plugin/skills/catherd/SKILL.md +2 -2
- package/src/domain/bars.ts +119 -0
- package/src/domain/calibration.ts +125 -0
- package/src/domain/catalog.ts +210 -23
- package/src/domain/failover.ts +14 -1
- package/src/domain/profile-rules.ts +108 -18
- package/src/domain/sources.ts +144 -0
- package/src/entry/catalog-command.ts +163 -8
- package/src/entry/init-command.ts +72 -2
- package/src/entry/mcp/lane-tools.ts +1 -1
- package/src/entry/mcp/server.ts +13 -3
- package/src/entry/mcp/setup-tools.ts +24 -2
- package/src/entry/profile-command.ts +10 -4
- package/src/entry/tui/commands.ts +20 -2
- package/src/entry/tui/effects.ts +50 -1
- package/src/entry/tui/fixtures.ts +31 -3
- package/src/entry/tui/profile-edits.ts +17 -9
- package/src/entry/tui/profile-tree.ts +2 -2
- package/src/entry/tui/state.ts +4 -0
- package/src/entry/tui/views/profile-actions.ts +9 -1
- package/src/entry/tui/views/profiles.tsx +81 -14
- package/src/entry/tui/views/save-dialog.tsx +12 -4
- package/src/infra/env.ts +2 -0
- package/src/infra/jev-client.ts +88 -40
- package/src/infra/paths.ts +2 -0
- package/src/infra/sources/arena.ts +87 -0
- package/src/infra/sources/artificial-analysis.ts +112 -0
- package/src/infra/sources/cache.ts +124 -0
- package/src/infra/sources/csv.ts +38 -0
- package/src/infra/sources/epoch.ts +82 -0
- package/src/infra/sources/http.ts +75 -0
- package/src/infra/sources/litellm.ts +58 -0
- package/src/infra/sources/models-dev.ts +59 -0
- package/src/infra/sources/openrouter-endpoints.ts +52 -0
- package/src/infra/sources/openrouter-models.ts +60 -0
- package/src/infra/sources/rows.ts +38 -0
- package/src/infra/sources/vectara.ts +69 -0
- package/src/infra/sources/zip.ts +48 -0
- package/src/services/catalog-refresh.ts +200 -0
- package/src/services/catalog-service.ts +83 -23
- package/src/services/credentials.ts +69 -0
- package/src/services/doctor-sources.ts +76 -0
- package/src/services/doctor.ts +10 -0
- package/src/services/jev-service.ts +14 -41
- package/src/services/ports.ts +6 -0
- package/src/services/profile-service.ts +8 -6
- package/src/services/provenance.ts +115 -0
- package/src/services/routing-service.ts +7 -2
- package/src/services/run-evidence.ts +86 -0
- package/src/services/source-derive.ts +349 -0
- package/src/services/source-sync.ts +305 -0
- package/src/services/standins.ts +204 -0
- package/src/services/treat-likes.ts +84 -0
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { z } from "zod";
|
|
4
|
+
import { CatherdError, isCatherdError } from "../domain/errors.ts";
|
|
5
|
+
import { log } from "../infra/log.ts";
|
|
6
|
+
import { configDir } from "../infra/paths.ts";
|
|
7
|
+
import { readVersioned, writeJsonAtomic } from "../infra/store.ts";
|
|
8
|
+
|
|
9
|
+
// Spec §10.4 and 1.2 §9: `<config>/credentials.json`, mode 600, holds catherd's own keys: Jev's
|
|
10
|
+
// (`typesafeApiKey`) and Artificial Analysis's (`artificialAnalysisApiKey`). The schema is a loose object,
|
|
11
|
+
// so a key one catherd does not know survives another's rewrite.
|
|
12
|
+
|
|
13
|
+
export const credentialsPath = (): string => join(configDir(), "credentials.json");
|
|
14
|
+
|
|
15
|
+
const CredentialsSchema = z.looseObject({
|
|
16
|
+
schema: z.literal(1).default(1),
|
|
17
|
+
typesafeApiKey: z.string().optional(),
|
|
18
|
+
artificialAnalysisApiKey: z.string().optional(),
|
|
19
|
+
});
|
|
20
|
+
export type CredentialField = "typesafeApiKey" | "artificialAnalysisApiKey";
|
|
21
|
+
|
|
22
|
+
/** How to repair a credentials file catherd cannot read: `init` alone refuses to overwrite it. */
|
|
23
|
+
const credentialsFix = (): string =>
|
|
24
|
+
`delete ${credentialsPath()} and run catherd init, or write it as {"schema": 1, "typesafeApiKey": "<your key>"}`;
|
|
25
|
+
const readCredentials = () =>
|
|
26
|
+
readVersioned(credentialsPath(), CredentialsSchema, 1, { fix: credentialsFix() });
|
|
27
|
+
|
|
28
|
+
/** A key saved in credentials.json, or why that file cannot be read (unparsable, newer schema). */
|
|
29
|
+
export function savedCredential(field: CredentialField): {
|
|
30
|
+
key: string | null;
|
|
31
|
+
problem: CatherdError | null;
|
|
32
|
+
} {
|
|
33
|
+
if (!existsSync(credentialsPath())) return { key: null, problem: null };
|
|
34
|
+
try {
|
|
35
|
+
return { key: readCredentials()[field]?.trim() || null, problem: null };
|
|
36
|
+
} catch (e) {
|
|
37
|
+
return {
|
|
38
|
+
key: null,
|
|
39
|
+
problem: isCatherdError(e)
|
|
40
|
+
? e
|
|
41
|
+
: new CatherdError("E_CONFIG_INVALID", String(e), { fix: credentialsFix() }),
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Keeps every other credential, and the file at mode 600 (spec §10.4). A missing file starts empty; an
|
|
48
|
+
* unreadable or newer-schema one is refused (it throws) rather than overwritten.
|
|
49
|
+
*/
|
|
50
|
+
export function saveCredential(field: CredentialField, key: string): void {
|
|
51
|
+
const cur: z.infer<typeof CredentialsSchema> = existsSync(credentialsPath())
|
|
52
|
+
? readCredentials()
|
|
53
|
+
: { schema: 1 };
|
|
54
|
+
writeJsonAtomic(credentialsPath(), { ...cur, schema: 1, [field]: key.trim() }, { mode: 0o600 });
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Spec 1.2 §9: `ARTIFICIAL_ANALYSIS_API_KEY`, else the saved key; null when neither has one. The key is
|
|
59
|
+
* optional, so a credentials file that cannot be read means no key here; it is logged.
|
|
60
|
+
*/
|
|
61
|
+
export function aaKey(env: Record<string, string | undefined> = process.env): string | null {
|
|
62
|
+
const fromEnv = env.ARTIFICIAL_ANALYSIS_API_KEY?.trim();
|
|
63
|
+
if (fromEnv) return fromEnv;
|
|
64
|
+
const { key, problem } = savedCredential("artificialAnalysisApiKey");
|
|
65
|
+
if (problem) log("warn", "sources", { error: `no Artificial Analysis key: ${problem.message}` });
|
|
66
|
+
return key;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export const saveAaKey = (key: string): void => saveCredential("artificialAnalysisApiKey", key);
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import type { Profile } from "../domain/profile.ts";
|
|
2
|
+
import { type StandInToConfirm, standInsToConfirm } from "../domain/profile-rules.ts";
|
|
3
|
+
import { loadCatalog } from "./catalog-service.ts";
|
|
4
|
+
import type { Check } from "./doctor-checks.ts";
|
|
5
|
+
import { runnableBackends } from "./profile-store.ts";
|
|
6
|
+
import { sourcesStatus, TTL_MS } from "./source-sync.ts";
|
|
7
|
+
|
|
8
|
+
const HOUR = 3_600_000;
|
|
9
|
+
const DAY = 24 * HOUR;
|
|
10
|
+
|
|
11
|
+
/** An age as doctor prints it: minutes under an hour, hours under two days, else days. */
|
|
12
|
+
export function ageText(ms: number): string {
|
|
13
|
+
if (ms < HOUR) return `${Math.max(1, Math.round(ms / 60_000))} min`;
|
|
14
|
+
if (ms < 2 * DAY) return `${Math.round(ms / HOUR)} h`;
|
|
15
|
+
return `${Math.round(ms / DAY)} d`;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** Spec 1.2 §6.1, §9: the stand-ins to confirm across `profiles`, one per canonical rung. */
|
|
19
|
+
export function standInsToConfirmIn(profiles: Profile[]): StandInToConfirm[] {
|
|
20
|
+
if (profiles.length === 0) return [];
|
|
21
|
+
const c = loadCatalog({ timings: false });
|
|
22
|
+
const seen = new Map<string, StandInToConfirm>();
|
|
23
|
+
for (const p of profiles)
|
|
24
|
+
for (const x of standInsToConfirm(p, c, runnableBackends()))
|
|
25
|
+
if (!seen.has(x.canonical)) seen.set(x.canonical, x);
|
|
26
|
+
return [...seen.values()];
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** `gpt-5.6-terra#high (repo_code, terminal, honesty)`, the rung and the dimensions a stand-in fills. */
|
|
30
|
+
const confirmText = (xs: StandInToConfirm[]): string =>
|
|
31
|
+
`stand-ins to confirm: ${xs.map((x) => `${x.canonical} (${x.dims.map((d) => d.dim).join(", ")})`).join(", ")}`;
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Spec 1.2 §9: the `sources` row: each source's age and last error, whether an Artificial Analysis key is
|
|
35
|
+
* set, its requests left today (`x-ratelimit-remaining` of its last answer), and the linked profiles'
|
|
36
|
+
* stand-ins to confirm (`confirm`, spec 1.2 §6.1). `info` before the first sync (routing uses the shipped
|
|
37
|
+
* scores); a warning when a source fails or has not been fetched for two TTLs (a day), since background
|
|
38
|
+
* syncs would have refreshed it. It reads the sync's state only: no request.
|
|
39
|
+
*/
|
|
40
|
+
export function sourcesCheck(now = Date.now(), confirm: StandInToConfirm[] = []): Check {
|
|
41
|
+
const s = sourcesStatus();
|
|
42
|
+
const shown = s.sources.filter((x) => x.source !== "artificial-analysis" || s.aaKey);
|
|
43
|
+
const keyless = s.sources.filter((x) => x.source !== "artificial-analysis");
|
|
44
|
+
const sameDay =
|
|
45
|
+
s.rateLimitAt !== null && s.rateLimitAt.slice(0, 10) === new Date(now).toISOString().slice(0, 10);
|
|
46
|
+
const left =
|
|
47
|
+
s.rateLimitRemaining === null
|
|
48
|
+
? ""
|
|
49
|
+
: sameDay
|
|
50
|
+
? `, ${s.rateLimitRemaining} requests left today`
|
|
51
|
+
: `, ${s.rateLimitRemaining} requests left on ${s.rateLimitAt?.slice(0, 10)}`;
|
|
52
|
+
const aa = `${s.aaKey ? `Artificial Analysis key set${left}` : "no Artificial Analysis key"}${confirm.length ? `; ${confirmText(confirm)}` : ""}`;
|
|
53
|
+
const base = { id: "sources", label: "sources" };
|
|
54
|
+
if (keyless.every((x) => x.fetchedAt === null && x.error === null))
|
|
55
|
+
return {
|
|
56
|
+
...base,
|
|
57
|
+
state: "info",
|
|
58
|
+
word: "not synced",
|
|
59
|
+
detail: `routing uses the shipped scores; ${aa}`,
|
|
60
|
+
fix: "catherd catalog sync",
|
|
61
|
+
};
|
|
62
|
+
const parts = shown.map(
|
|
63
|
+
(x) =>
|
|
64
|
+
`${x.source} ${x.fetchedAt ? ageText(now - Date.parse(x.fetchedAt)) : "never"}${x.error ? ` (${x.error})` : ""}`,
|
|
65
|
+
);
|
|
66
|
+
const failing = shown.some((x) => x.error !== null);
|
|
67
|
+
const stale = keyless.some((x) => x.fetchedAt === null || now - Date.parse(x.fetchedAt) > 2 * TTL_MS);
|
|
68
|
+
const warn = failing || stale;
|
|
69
|
+
return {
|
|
70
|
+
...base,
|
|
71
|
+
state: warn ? "warn" : "ok",
|
|
72
|
+
word: failing ? "failing" : stale ? "stale" : "fresh",
|
|
73
|
+
detail: `${parts.join(", ")}; ${aa}`,
|
|
74
|
+
...(warn ? { fix: "catherd catalog sync --force" } : {}),
|
|
75
|
+
};
|
|
76
|
+
}
|
package/src/services/doctor.ts
CHANGED
|
@@ -9,6 +9,7 @@ import { linkedProfiles } from "./agent-links.ts";
|
|
|
9
9
|
import { accessChecks } from "./doctor-access.ts";
|
|
10
10
|
import { backendChecks, usedBackends } from "./doctor-backends.ts";
|
|
11
11
|
import { type PushProbe, pushCheck } from "./doctor-push.ts";
|
|
12
|
+
import { sourcesCheck, standInsToConfirmIn } from "./doctor-sources.ts";
|
|
12
13
|
import {
|
|
13
14
|
agentsCheck,
|
|
14
15
|
type Check,
|
|
@@ -190,6 +191,15 @@ export async function doctor(d: DoctorDeps): Promise<DoctorReport> {
|
|
|
190
191
|
}
|
|
191
192
|
}
|
|
192
193
|
|
|
194
|
+
// spec 1.2 §9: the public sources' ages and errors, the Artificial Analysis key, and the active and linked
|
|
195
|
+
// profiles' stand-ins to confirm
|
|
196
|
+
const mine = [...(active ? [active] : []), ...profiles.filter((p) => p.name !== active?.name)];
|
|
197
|
+
checks.push(
|
|
198
|
+
guarded("sources", "sources", "catherd catalog sync --force", () =>
|
|
199
|
+
sourcesCheck(Date.now(), standInsToConfirmIn(mine)),
|
|
200
|
+
),
|
|
201
|
+
);
|
|
202
|
+
|
|
193
203
|
checks.push(pluginCheck(d.version));
|
|
194
204
|
|
|
195
205
|
checks.push(
|
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
import { existsSync } from "node:fs";
|
|
2
1
|
import { join } from "node:path";
|
|
3
|
-
import {
|
|
4
|
-
import { CatherdError, isCatherdError } from "../domain/errors.ts";
|
|
2
|
+
import type { CatherdError } from "../domain/errors.ts";
|
|
5
3
|
import type { RouteSource } from "../domain/route.ts";
|
|
6
4
|
import {
|
|
7
5
|
canonicalJson,
|
|
@@ -17,45 +15,28 @@ import {
|
|
|
17
15
|
import { assetPath } from "../infra/assets.ts";
|
|
18
16
|
import { type JevTransport, jevRequest } from "../infra/jev-client.ts";
|
|
19
17
|
import { addSecret, log } from "../infra/log.ts";
|
|
20
|
-
import {
|
|
21
|
-
import {
|
|
18
|
+
import { appendJsonl, ensureJsonlHeader, readJsonl, readVersioned } from "../infra/store.ts";
|
|
19
|
+
import { aaKey, credentialsPath, saveCredential, savedCredential } from "./credentials.ts";
|
|
20
|
+
|
|
21
|
+
export { credentialsPath };
|
|
22
22
|
|
|
23
23
|
let file: JevFile | null = null;
|
|
24
24
|
/** catalog/jev.json: the question sets and their rules (spec §5.5). */
|
|
25
25
|
export const jevQuestions = (): JevFile =>
|
|
26
26
|
(file ??= readVersioned(assetPath("catalog/jev.json"), JevFileSchema, 1));
|
|
27
27
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
});
|
|
33
|
-
|
|
34
|
-
/** Registers the saved Jev key with the redactor, for a process that reads logs without ever calling Jev. */
|
|
28
|
+
/**
|
|
29
|
+
* Registers the saved keys (Jev's and Artificial Analysis's) with the redactor, for a process that reads
|
|
30
|
+
* logs without ever using them.
|
|
31
|
+
*/
|
|
35
32
|
export function registerSavedSecrets(): void {
|
|
36
33
|
addSecret(jevKey());
|
|
34
|
+
addSecret(aaKey());
|
|
37
35
|
}
|
|
38
36
|
|
|
39
|
-
/** How to repair a credentials file catherd cannot read: `init` alone refuses to overwrite it. */
|
|
40
|
-
const credentialsFix = (): string =>
|
|
41
|
-
`delete ${credentialsPath()} and run catherd init, or write it as {"schema": 1, "typesafeApiKey": "<your key>"}`;
|
|
42
|
-
const readCredentials = () =>
|
|
43
|
-
readVersioned(credentialsPath(), CredentialsSchema, 1, { fix: credentialsFix() });
|
|
44
|
-
|
|
45
37
|
/** The key saved in credentials.json, or why that file cannot be read (unparsable, newer schema). */
|
|
46
|
-
export
|
|
47
|
-
|
|
48
|
-
try {
|
|
49
|
-
return { key: readCredentials().typesafeApiKey?.trim() || null, problem: null };
|
|
50
|
-
} catch (e) {
|
|
51
|
-
return {
|
|
52
|
-
key: null,
|
|
53
|
-
problem: isCatherdError(e)
|
|
54
|
-
? e
|
|
55
|
-
: new CatherdError("E_CONFIG_INVALID", String(e), { fix: credentialsFix() }),
|
|
56
|
-
};
|
|
57
|
-
}
|
|
58
|
-
}
|
|
38
|
+
export const savedJevKey = (): { key: string | null; problem: CatherdError | null } =>
|
|
39
|
+
savedCredential("typesafeApiKey");
|
|
59
40
|
|
|
60
41
|
/**
|
|
61
42
|
* Spec §5.5: `TYPESAFE_API_KEY`, else `<config>/credentials.json`; null when neither has one. Jev is
|
|
@@ -70,16 +51,8 @@ export function jevKey(): string | null {
|
|
|
70
51
|
return key;
|
|
71
52
|
}
|
|
72
53
|
|
|
73
|
-
/**
|
|
74
|
-
|
|
75
|
-
* unreadable or newer-schema one is refused (it throws) rather than overwritten.
|
|
76
|
-
*/
|
|
77
|
-
export function saveJevKey(key: string): void {
|
|
78
|
-
const cur: z.infer<typeof CredentialsSchema> = existsSync(credentialsPath())
|
|
79
|
-
? readCredentials()
|
|
80
|
-
: { schema: 1 };
|
|
81
|
-
writeJsonAtomic(credentialsPath(), { ...cur, schema: 1, typesafeApiKey: key.trim() }, { mode: 0o600 });
|
|
82
|
-
}
|
|
54
|
+
/** Saves the Jev key in credentials.json, keeping every other credential (`saveCredential`). */
|
|
55
|
+
export const saveJevKey = (key: string): void => saveCredential("typesafeApiKey", key);
|
|
83
56
|
|
|
84
57
|
/** A key works when Jev lists its models: no inference, and no dependence on a question set. */
|
|
85
58
|
export async function testJevKey(key: string, o: JevTransport = {}): Promise<boolean> {
|
package/src/services/ports.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { Provenance } from "./provenance.ts";
|
|
1
2
|
import type { Budget } from "../domain/budget.ts";
|
|
2
3
|
import type { SessionEnv } from "../infra/claude-session.ts";
|
|
3
4
|
import type { BillingMode } from "../domain/cost.ts";
|
|
@@ -8,6 +9,7 @@ import type { Role } from "../domain/roles.ts";
|
|
|
8
9
|
import type { Change, ProfilePatch } from "../domain/profile.ts";
|
|
9
10
|
import type { Issue } from "../domain/profile-rules.ts";
|
|
10
11
|
import type { RouteJev, RouteSource } from "../domain/route.ts";
|
|
12
|
+
import type { SyncReport } from "./source-sync.ts";
|
|
11
13
|
|
|
12
14
|
/** What the run lifecycle reads from a profile. Rungs are `backend:model#effort`; a role's in ladder order. */
|
|
13
15
|
export interface ProfileView {
|
|
@@ -86,6 +88,8 @@ export interface RouteAnswer {
|
|
|
86
88
|
/** the Jev question set asked, and its summed probabilities, for outcomes.jsonl (spec §5.6) */
|
|
87
89
|
questionSet: string | null;
|
|
88
90
|
jev: RouteJev | null;
|
|
91
|
+
/** spec 1.2 §5.3, §8: the chosen rung's thresholds, values, sources, facts and run evidence */
|
|
92
|
+
provenance?: Provenance;
|
|
89
93
|
}
|
|
90
94
|
|
|
91
95
|
export type { Verdict };
|
|
@@ -132,4 +136,6 @@ export interface Deps {
|
|
|
132
136
|
/** the Claude Code session this process serves (spec §3.3), from its environment; null outside one */
|
|
133
137
|
session: SessionEnv | null;
|
|
134
138
|
now: () => number;
|
|
139
|
+
/** spec 1.2 §3.2 `catalog_sync`; default: the real sync (tests inject one that never reaches the network) */
|
|
140
|
+
sync?: (o: { force: boolean }) => Promise<SyncReport>;
|
|
135
141
|
}
|
|
@@ -15,7 +15,7 @@ import {
|
|
|
15
15
|
type ProfilePatch,
|
|
16
16
|
resolveProfile,
|
|
17
17
|
} from "../domain/profile.ts";
|
|
18
|
-
import { type Validation, validateProfile } from "../domain/profile-rules.ts";
|
|
18
|
+
import { repairs, type Validation, validateProfile } from "../domain/profile-rules.ts";
|
|
19
19
|
import { ROLES } from "../domain/roles.ts";
|
|
20
20
|
import { withFileLockSync } from "../infra/filelock.ts";
|
|
21
21
|
import { configDir } from "../infra/paths.ts";
|
|
@@ -85,10 +85,12 @@ const unsaved = (v: Validation): Saved => ({
|
|
|
85
85
|
export const CHANGED_ON_DISK = "profile";
|
|
86
86
|
|
|
87
87
|
/**
|
|
88
|
-
* Spec §7.3 `patch` (profile_set, `catherd profile set
|
|
89
|
-
*
|
|
90
|
-
*
|
|
91
|
-
*
|
|
88
|
+
* Spec §7.3 `patch` (profile_set, `catherd profile set`, the TUI's save): validates first and writes
|
|
89
|
+
* nothing when invalid, unless the save repairs the profile (spec 1.2 §6.2): it removes at least one error
|
|
90
|
+
* the stored profile had and adds none. Then it saves, and the result's `errors` are those still open. A
|
|
91
|
+
* profile that does not exist yet starts from the default profile. With `expect` (the profile as a preview
|
|
92
|
+
* read it), it writes nothing when the profile under the lock is no longer that one: the patch would land on
|
|
93
|
+
* values the preview never showed.
|
|
92
94
|
*/
|
|
93
95
|
export function patchProfile(
|
|
94
96
|
name: string | undefined,
|
|
@@ -112,7 +114,7 @@ export function patchProfile(
|
|
|
112
114
|
const after = applyPatch(before, patch);
|
|
113
115
|
const resolved = resolveProfile(after, n);
|
|
114
116
|
const v = validate(after, n);
|
|
115
|
-
if (v.errors.length) return unsaved(v);
|
|
117
|
+
if (v.errors.length && !repairs(profileExists(n) ? validate(before, n) : null, v)) return unsaved(v);
|
|
116
118
|
const diff = diffProfiles(resolveProfile(before, n), resolved);
|
|
117
119
|
return { saved: true, ...v, diff, ...saveAndLink(n, after) };
|
|
118
120
|
});
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { type Catalog, type Confidence, DIMS, type Dim, rungInfo, scoresOf } from "../domain/catalog.ts";
|
|
2
|
+
import { type BillingMode, type Cost, costOf, DEFAULT_BILLING } from "../domain/cost.ts";
|
|
3
|
+
import type { Difficulty, Kind } from "../domain/lane.ts";
|
|
4
|
+
import { type EvidenceTable, evidenceLine, evidenceOf } from "./run-evidence.ts";
|
|
5
|
+
|
|
6
|
+
/** Spec 1.2 §5.3: one value routing uses for a rung, with where it came from. */
|
|
7
|
+
export interface ValueUsed {
|
|
8
|
+
dim: Dim;
|
|
9
|
+
value: number;
|
|
10
|
+
confidence: Confidence;
|
|
11
|
+
/** the source a synced value came from; `shipped` for a hand-typed one, `override` for the user's */
|
|
12
|
+
source: string;
|
|
13
|
+
benchmark: string;
|
|
14
|
+
date: string;
|
|
15
|
+
url: string;
|
|
16
|
+
/** catherd's guess for this rung: lent by a treat-like or an inferred stand-in (spec 1.2 §6.1) */
|
|
17
|
+
inferred: boolean;
|
|
18
|
+
/** the rung the value belongs to, when it is not this one */
|
|
19
|
+
from: string | null;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** Spec 1.2 §5.3: one threshold of the lane's bar, the value used against it, and whether it clears. */
|
|
23
|
+
export interface Threshold {
|
|
24
|
+
dim: Dim;
|
|
25
|
+
min: number;
|
|
26
|
+
used: ValueUsed | null;
|
|
27
|
+
clears: boolean;
|
|
28
|
+
/** where the default threshold came from (`barsWhy`) */
|
|
29
|
+
why: string | null;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Spec 1.2 §5.3, §8: what `route` reports about the rung it chose. */
|
|
33
|
+
export interface Provenance {
|
|
34
|
+
rung: string;
|
|
35
|
+
canonical: string;
|
|
36
|
+
/** the lane's bar, threshold by threshold; empty when the route read no bar (the default rung) */
|
|
37
|
+
thresholds: Threshold[];
|
|
38
|
+
/** every value the rung has */
|
|
39
|
+
values: ValueUsed[];
|
|
40
|
+
/** facts that never carry a bar (spec 1.2 §4.1): `<source>.<field>` → value */
|
|
41
|
+
speed: Record<string, number>;
|
|
42
|
+
cost: Cost;
|
|
43
|
+
/** spec 1.2 §8: its runs for the lane's kind (or every kind) and over every kind; never used for routing */
|
|
44
|
+
evidence: { kind: string | null; all: string | null };
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Spec 1.2 §5.3: each value a canonical rung has, with its confidence and source, borrowed ones marked. */
|
|
48
|
+
export function valuesUsed(c: Catalog, canonical: string): ValueUsed[] {
|
|
49
|
+
const s = scoresOf(c, canonical);
|
|
50
|
+
if (!s) return [];
|
|
51
|
+
return DIMS.flatMap((dim) => {
|
|
52
|
+
const r = s.records[dim];
|
|
53
|
+
if (!r) return [];
|
|
54
|
+
const from = s.borrowed.includes(dim) ? s.via : (s.standIns[dim] ?? null);
|
|
55
|
+
return [
|
|
56
|
+
{
|
|
57
|
+
dim,
|
|
58
|
+
value: r.value,
|
|
59
|
+
confidence: r.confidence,
|
|
60
|
+
source: r.source ?? "shipped",
|
|
61
|
+
benchmark: `${r.benchmark} ${r.version}`,
|
|
62
|
+
date: r.date,
|
|
63
|
+
url: r.url,
|
|
64
|
+
inferred: from !== null,
|
|
65
|
+
from,
|
|
66
|
+
},
|
|
67
|
+
];
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Spec 1.2 §5.3: the chosen rung's thresholds for the lane's kind and difficulty, its values and facts. */
|
|
72
|
+
export function provenanceOf(
|
|
73
|
+
c: Catalog,
|
|
74
|
+
rung: string,
|
|
75
|
+
kind: Kind | null,
|
|
76
|
+
difficulty: Difficulty | null,
|
|
77
|
+
billing: Partial<Record<string, BillingMode>>,
|
|
78
|
+
evidence: EvidenceTable,
|
|
79
|
+
): Provenance {
|
|
80
|
+
const info = rungInfo(c, rung);
|
|
81
|
+
const values = valuesUsed(c, info.canonical);
|
|
82
|
+
const bar = kind && difficulty ? c.bars[kind][difficulty] : {};
|
|
83
|
+
const thresholds: Threshold[] = DIMS.flatMap((dim) => {
|
|
84
|
+
const min = bar[dim];
|
|
85
|
+
if (min === undefined || !kind || !difficulty) return [];
|
|
86
|
+
const used = values.find((v) => v.dim === dim) ?? null;
|
|
87
|
+
return [
|
|
88
|
+
{
|
|
89
|
+
dim,
|
|
90
|
+
min,
|
|
91
|
+
used,
|
|
92
|
+
clears: used !== null && used.value >= min,
|
|
93
|
+
why: c.barsWhy[dim]?.[difficulty] ?? null,
|
|
94
|
+
},
|
|
95
|
+
];
|
|
96
|
+
});
|
|
97
|
+
return {
|
|
98
|
+
rung,
|
|
99
|
+
canonical: info.canonical,
|
|
100
|
+
thresholds,
|
|
101
|
+
values,
|
|
102
|
+
speed: info.family?.speed ?? {},
|
|
103
|
+
cost: costOf(info.family, info.parsed.effort, billing[info.key] ?? DEFAULT_BILLING[info.key]),
|
|
104
|
+
evidence: {
|
|
105
|
+
kind: evidenceLine(evidenceOf(evidence, info.canonical, kind)),
|
|
106
|
+
all: evidenceLine(evidenceOf(evidence, info.canonical)),
|
|
107
|
+
},
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** A value as one line: `repo_code 66.6 (adjacent, shipped DeepSWE 1.1, 2026-09-22)`, marking a guess. */
|
|
112
|
+
export function valueWords(v: ValueUsed): string {
|
|
113
|
+
const lent = v.inferred ? `, inferred from ${v.from}` : "";
|
|
114
|
+
return `${v.dim} ${v.value} (${v.confidence}, ${v.source} ${v.benchmark}, ${v.date}${lent})`;
|
|
115
|
+
}
|
|
@@ -24,6 +24,8 @@ import { log } from "../infra/log.ts";
|
|
|
24
24
|
import { catalogQuery, freshenDiscovery, loadCatalog } from "./catalog-service.ts";
|
|
25
25
|
import { type Asked, askJev, type JevOpts, jevQuestions, logJev } from "./jev-service.ts";
|
|
26
26
|
import type { ProfileView, RouteAnswer, RouteRequest, RoutingPort, Verdict } from "./ports.ts";
|
|
27
|
+
import { provenanceOf } from "./provenance.ts";
|
|
28
|
+
import { runEvidence } from "./run-evidence.ts";
|
|
27
29
|
|
|
28
30
|
function routingProfile(v: ProfileView, role: Role, spentFraction: number): RoutingProfile {
|
|
29
31
|
const rc = v.roles[role];
|
|
@@ -87,8 +89,10 @@ async function route(req: RouteRequest, o: RoutingOpts): Promise<RouteAnswer> {
|
|
|
87
89
|
await freshenWithin(p.role.rungs, req.repo, o.discoveryBudgetMs ?? DISCOVERY_BUDGET_MS);
|
|
88
90
|
const c = loadCatalog({ repo: req.repo });
|
|
89
91
|
const fallback = () => defaultLadder(c, p, req.role);
|
|
90
|
-
if (req.laneText === null || candidates(c, p, req.role).length <= 1)
|
|
91
|
-
|
|
92
|
+
if (req.laneText === null || candidates(c, p, req.role).length <= 1) {
|
|
93
|
+
const out = answer(fallback(), "default", null, null, null, null);
|
|
94
|
+
return { ...out, provenance: provenanceOf(c, out.rung, null, null, p.billing, runEvidence(c)) };
|
|
95
|
+
}
|
|
92
96
|
const lane = parseLaneHeader(req.laneText);
|
|
93
97
|
let asked: Asked | null = null;
|
|
94
98
|
let judged: ReturnType<typeof judgeRoute> | null = null;
|
|
@@ -120,6 +124,7 @@ async function route(req: RouteRequest, o: RoutingOpts): Promise<RouteAnswer> {
|
|
|
120
124
|
? answer(select(c, p, req.role, kind, lane.difficulty), "lane", kind, lane.difficulty, asked, jev)
|
|
121
125
|
: answer(fallback(), "default", null, null, asked, jev);
|
|
122
126
|
}
|
|
127
|
+
out.provenance = provenanceOf(c, out.rung, out.kind, out.difficulty, p.billing, runEvidence(c));
|
|
123
128
|
if (asked) {
|
|
124
129
|
logJev(req.runDir, {
|
|
125
130
|
...asked.meta,
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { type Catalog, rungInfo } from "../domain/catalog.ts";
|
|
2
|
+
import type { Kind } from "../domain/lane.ts";
|
|
3
|
+
import { routeAt } from "../domain/route.ts";
|
|
4
|
+
import { replyVerdict } from "./milestones.ts";
|
|
5
|
+
import { listRuns, readAgentRuns, readRecords, readRoutes } from "./run-store.ts";
|
|
6
|
+
|
|
7
|
+
/** Spec 1.2 §8: what catherd's own runs say about a rung, for one kind or every kind (`*`). */
|
|
8
|
+
export interface Evidence {
|
|
9
|
+
/** lanes routed to it, or climbed onto it */
|
|
10
|
+
lanes: number;
|
|
11
|
+
/** lanes that climbed off it */
|
|
12
|
+
climbed: number;
|
|
13
|
+
partial: number;
|
|
14
|
+
blocked: number;
|
|
15
|
+
refused: number;
|
|
16
|
+
/** verifier FAILs it gave, as a verifier (plan 14 Ruling 15) */
|
|
17
|
+
fails: number;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** `<canonical rung>|<kind or *>` → its evidence. Shown by route, catalog list and the TUI; never routes. */
|
|
21
|
+
export type EvidenceTable = Record<string, Evidence>;
|
|
22
|
+
|
|
23
|
+
const VERDICT_FAIL = /^VERDICT: FAIL\b/;
|
|
24
|
+
const zero = (): Evidence => ({ lanes: 0, climbed: 0, partial: 0, blocked: 0, refused: 0, fails: 0 });
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Spec 1.2 §8: for each rung and kind, from the run records of every repo on this machine: the lanes routed or
|
|
28
|
+
* climbed onto it, the climbs off it, its `partial`, `blocked` and `refused` replies (under the kind its lane
|
|
29
|
+
* was routed as when the dispatch started), and the verifier FAILs it gave. Every count also goes under `*`.
|
|
30
|
+
* It never changes scores, bars or routing (1.3's, once there is data).
|
|
31
|
+
*/
|
|
32
|
+
export function runEvidence(c: Catalog): EvidenceTable {
|
|
33
|
+
const out: EvidenceTable = {};
|
|
34
|
+
const canonicalOf = (rung: string): string | null => {
|
|
35
|
+
try {
|
|
36
|
+
return rungInfo(c, rung).canonical;
|
|
37
|
+
} catch {
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
};
|
|
41
|
+
const bump = (rung: string, kind: Kind | null, field: keyof Evidence) => {
|
|
42
|
+
const canonical = canonicalOf(rung);
|
|
43
|
+
if (!canonical) return;
|
|
44
|
+
for (const k of kind ? [kind, "*"] : ["*"]) (out[`${canonical}|${k}`] ??= zero())[field] += 1;
|
|
45
|
+
};
|
|
46
|
+
for (const run of listRuns().runs) {
|
|
47
|
+
const routes = readRoutes(run);
|
|
48
|
+
// a lane counts once per rung it was on, under the kind of the row that put it there
|
|
49
|
+
const seen = new Set<string>();
|
|
50
|
+
for (const r of routes) {
|
|
51
|
+
const key = `${r.lane} ${r.rung} ${r.kind ?? "*"}`;
|
|
52
|
+
if (!seen.has(key)) {
|
|
53
|
+
seen.add(key);
|
|
54
|
+
bump(r.rung, r.kind, "lanes");
|
|
55
|
+
}
|
|
56
|
+
if (r.source === "climb" && r.from && r.from !== r.rung) bump(r.from, r.kind, "climbed");
|
|
57
|
+
}
|
|
58
|
+
for (const rec of readRecords(run).records) {
|
|
59
|
+
const kind = rec.lane ? (routeAt(routes, rec.lane, rec.startedAt)?.kind ?? null) : null;
|
|
60
|
+
if (rec.replyStatus === "partial" || rec.replyStatus === "blocked" || rec.replyStatus === "refused")
|
|
61
|
+
bump(rec.rung, kind, rec.replyStatus);
|
|
62
|
+
if (rec.role === "verifier" && rec.status === "ok" && VERDICT_FAIL.test(replyVerdict(run, rec) ?? ""))
|
|
63
|
+
bump(rec.rung, null, "fails");
|
|
64
|
+
}
|
|
65
|
+
for (const a of readAgentRuns(run))
|
|
66
|
+
if (a.role === "verifier" && a.status === "failed") bump(a.rung, null, "fails");
|
|
67
|
+
}
|
|
68
|
+
return out;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** A rung's evidence for `kind`, else over every kind; null when catherd never ran it. */
|
|
72
|
+
export function evidenceOf(t: EvidenceTable, canonical: string, kind: Kind | null = null): Evidence | null {
|
|
73
|
+
return (kind ? t[`${canonical}|${kind}`] : undefined) ?? t[`${canonical}|*`] ?? null;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Spec 1.2 §8's words: "12 lanes, 2 climbed, 1 partial"; null when there is nothing to say. */
|
|
77
|
+
export function evidenceLine(e: Evidence | null): string | null {
|
|
78
|
+
if (!e) return null;
|
|
79
|
+
const parts = [`${e.lanes} ${e.lanes === 1 ? "lane" : "lanes"}`];
|
|
80
|
+
if (e.climbed) parts.push(`${e.climbed} climbed`);
|
|
81
|
+
if (e.partial) parts.push(`${e.partial} partial`);
|
|
82
|
+
if (e.blocked) parts.push(`${e.blocked} blocked`);
|
|
83
|
+
if (e.refused) parts.push(`${e.refused} refused`);
|
|
84
|
+
if (e.fails) parts.push(`${e.fails} verifier ${e.fails === 1 ? "FAIL" : "FAILs"}`);
|
|
85
|
+
return parts.join(", ");
|
|
86
|
+
}
|