catherd-cli 0.2.1 → 1.0.0

package/src/services/catalog-service.ts

@@ -0,0 +1,382 @@
+import { existsSync } from "node:fs";
+import { dirname, join } from "node:path";
+import type { DiscoveredModel } from "../adapters/backend.ts";
+import { discovered, listingRepo, readDiscovery, writeDiscovery } from "../adapters/discovery.ts";
+import { adapterFor } from "../adapters/registry.ts";
+import "../adapters/all.ts";
+import {
+  buildCatalog,
+  type Catalog,
+  CanonicalRung,
+  capableFor,
+  DIMS,
+  type Family,
+  type ModelsFile,
+  ModelsFileSchema,
+  type Override,
+  OverrideSchema,
+  rungInfo,
+  type ScoresFile,
+  ScoresFileSchema,
+  scoresOf,
+} from "../domain/catalog.ts";
+import { costOf, DEFAULT_BILLING, type BillingMode } from "../domain/cost.ts";
+import { CatherdError, errorMessage } from "../domain/errors.ts";
+import { ADAPTER_IDS, tryParseRung } from "../domain/ids.ts";
+import type { Kind } from "../domain/lane.ts";
+import { routeAt } from "../domain/route.ts";
+import { ROLES, type Role } from "../domain/roles.ts";
+import { median } from "../domain/util.ts";
+import { assetPath } from "../infra/assets.ts";
+import { withFileLock } from "../infra/filelock.ts";
+import { log } from "../infra/log.ts";
+import { configDir } from "../infra/paths.ts";
+import { ensurePrivateDir, readVersioned, writeJsonAtomic } from "../infra/store.ts";
+import type { CatalogFilter } from "./ports.ts";
+import { listRuns, readAgentRuns, readRecords, readRoutes } from "./run-store.ts";
+
+const DAY_MS = 24 * 3_600_000;
+/** Spec §5.2: `secs_per_task` counts once a rung has this many of the user's own runs. */
+const MIN_SAMPLES = 5;
+
+let models: ModelsFile | null = null;
+let scores: ScoresFile | null = null;
+const shippedModels = (): ModelsFile =>
+  (models ??= readVersioned(assetPath("catalog/models.json"), ModelsFileSchema, 1));
+const shippedScores = (): ScoresFile =>
+  (scores ??= readVersioned(assetPath("catalog/scores.json"), ScoresFileSchema, 1));
+
+export const overridePath = (): string => join(configDir(), "catalog.override.json");
+
+function readOverride(): Override {
+  const file = overridePath();
+  return existsSync(file) ? readVersioned(file, OverrideSchema, 1) : OverrideSchema.parse({});
+}
+
+/**
+ * Every backend's last listing (spec §3.5 `<data>/discovery/<backend>.json`); for a backend listed per
+ * repository, its listing in `repo` (none without one).
+ */
+function listedModels(repo?: string): Catalog["listed"] {
+  const out: Catalog["listed"] = {};
+  for (const id of ADAPTER_IDS) {
+    const d = readDiscovery(id, listingRepo(id, repo));
+    if (d) out[id] = { fetchedAt: d.fetchedAt, models: d.models };
+  }
+  return out;
+}
+
+/**
+ * Spec §5.2: the median seconds of the user's own successful runs per canonical rung and lane kind
+ * (`|*` over every kind), kept only with at least MIN_SAMPLES runs. A run counts under the kind of its
+ * lane's route in force when it started; a run with none counts only under `|*`. Native `claude:`
+ * subagent runs (agents.jsonl, from record_agent_run) count too; one that names no lane counts only
+ * under `|*`.
+ */
+export function measuredSecs(base: Catalog): Catalog["secs"] {
+  const groups = new Map<string, number[]>();
+  const add = (k: string, secs: number) => groups.set(k, [...(groups.get(k) ?? []), secs]);
+  const count = (
+    routes: ReturnType<typeof readRoutes>,
+    rung: string,
+    secs: number,
+    lane: string | null,
+    startedAt: string,
+  ) => {
+    let canonical: string;
+    try {
+      canonical = rungInfo(base, rung).canonical;
+    } catch {
+      return;
+    }
+    // the kind the lane was routed as when this run started, not its newest route
+    const kind: Kind | null = lane ? (routeAt(routes, lane, startedAt)?.kind ?? null) : null;
+    add(`${canonical}|*`, secs);
+    if (kind) add(`${canonical}|${kind}`, secs);
+  };
+  for (const run of listRuns().runs) {
+    const routes = readRoutes(run);
+    for (const r of readRecords(run).records)
+      if (r.status === "ok") count(routes, r.rung, r.secs, r.lane, r.startedAt);
+    for (const a of readAgentRuns(run)) {
+      if (a.status !== "ok" || a.secs === null) continue;
+      // a row is written when the subagent ends; it started `secs` earlier
+      const lane = typeof a.lane === "string" ? a.lane : null;
+      const end = Date.parse(a.at);
+      // a duration past the Date range (record_agent_run takes any) keeps the end, not a RangeError
+      const start = new Date(end - a.secs * 1000);
+      const startedAt = Number.isNaN(start.getTime()) ? a.at : start.toISOString();
+      count(routes, a.rung, a.secs, lane, startedAt);
+    }
+  }
+  const secs: Catalog["secs"] = {};
+  for (const [k, xs] of groups) if (xs.length >= MIN_SAMPLES) secs[k] = median(xs) as number;
+  return secs;
+}
+
+/**
+ * The shipped catalog with every listing (in `repo` for a backend listed per repository), the user's
+ * override and, unless `timings: false`, own timings.
+ */
+export function loadCatalog(o: { timings?: boolean; repo?: string } = {}): Catalog {
+  const base = buildCatalog({
+    models: shippedModels(),
+    scores: shippedScores(),
+    override: readOverride(),
+    listed: listedModels(o.repo),
+  });
+  return o.timings === false ? base : { ...base, secs: measuredSecs(base) };
+}
+
+export interface Refreshed {
+  backend: string;
+  models: number;
+  fetchedAt: string | null;
+  error?: string;
+  /** the command that fixes `error`, when the backend's probe names one */
+  fix?: string;
+}
+
+/** Spec §5.2: `init`, `doctor` and `catherd catalog refresh` list every backend's models now. */
+export async function refreshDiscovery(
+  o: { backends?: string[]; now?: number; repo?: string } = {},
+): Promise<Refreshed[]> {
+  const out: Refreshed[] = [];
+  for (const id of o.backends ?? ADAPTER_IDS) {
+    const adapter = adapterFor(id);
+    if (!adapter) continue;
+    const repo = listingRepo(id, o.repo);
+    let listed: DiscoveredModel[] = [];
+    try {
+      listed = await adapter.listModels(repo);
+    } catch (e) {
+      out.push({
+        backend: id,
+        models: 0,
+        fetchedAt: readDiscovery(id, repo)?.fetchedAt ?? null,
+        error: String(e),
+      });
+      continue;
+    }
+    if (listed.length === 0) {
+      // a CLI that is missing (or not ready) lists nothing: say why, and keep only a listing that exists
+      const problem = (await adapter.probe().catch(() => null))?.problems[0];
+      const fetchedAt = readDiscovery(id, repo)?.fetchedAt ?? null;
+      out.push({
+        backend: id,
+        models: 0,
+        fetchedAt,
+        error: `${problem?.message ?? "listed no models"}${fetchedAt === null ? "" : "; the previous listing is kept"}`,
+        ...(problem?.fix ? { fix: problem.fix } : {}),
+      });
+      continue;
+    }
+    const f = writeDiscovery(id, listed, o.now, repo);
+    out.push({ backend: id, models: f.models.length, fetchedAt: f.fetchedAt });
+  }
+  return out;
+}
+
+const tried = new Map<string, number>();
+/** Forget which backends (and repositories) this process already tried to list (tests). */
+export const resetFreshen = (): void => tried.clear();
+
+/**
+ * Spec §5.2: `route` lists a backend again at most daily. A backend this process failed to list is not
+ * tried again for an hour, so a missing or wedged CLI never slows every route. Due backends are listed
+ * in parallel, so the slowest listing bounds the wait, not their sum. A backend listed per repository is
+ * listed, cached and backed off in `repo`.
+ */
+export async function freshenDiscovery(rungs: string[], now = Date.now(), repo?: string): Promise<void> {
+  const backends = new Set<string>();
+  for (const r of rungs) {
+    const b = tryParseRung(r)?.backend;
+    if (b) backends.add(b === "claude" ? "claude-code" : b); // validate reports a bad rung
+  }
+  const due: { backend: string; listing: Promise<unknown> }[] = [];
+  for (const b of backends) {
+    const adapter = adapterFor(b);
+    const where = listingRepo(b, repo);
+    const key = where === undefined ? b : `${b} ${where}`;
+    if (!adapter || now - (tried.get(key) ?? Number.NEGATIVE_INFINITY) < 3_600_000) continue;
+    const cached = readDiscovery(b, where);
+    if (cached && now - Date.parse(cached.fetchedAt) < DAY_MS) continue;
+    tried.set(key, now);
+    due.push({
+      backend: b,
+      listing: discovered(b, () => adapter.listModels(where), { maxAgeMs: DAY_MS, now, repo: where }),
+    });
+  }
+  // a failed listing keeps the last one (and the hour's backoff above); the log says why it failed
+  const settled = await Promise.allSettled(due.map((d) => d.listing));
+  settled.forEach((r, i) => {
+    if (r.status === "rejected")
+      log("debug", "discovery", { backend: due[i]?.backend, error: errorMessage(r.reason) });
+  });
+}
+
+/** A rung given as `backend:model#effort` or as a canonical `model#effort`, in canonical form. */
+function canonicalOf(c: Catalog, rung: string): string {
+  let canonical = rung;
+  try {
+    if (rung.includes(":")) canonical = rungInfo(c, rung).canonical;
+  } catch {
+    // a rung the catalog cannot resolve stays as given; the canonical check below refuses it
+  }
+  // a backend's own model id without its backend: the family that backend runs it for names it
+  const hash = canonical.lastIndexOf("#");
+  const model = canonical.slice(0, hash);
+  if (hash > 0 && !c.families.some((f) => f.id === model)) {
+    const family = c.families.find((f) => Object.values(f.on).some((m) => m?.id === model));
+    if (family) canonical = `${family.id}${canonical.slice(hash)}`;
+  }
+  if (!CanonicalRung.safeParse(canonical).success)
+    throw new CatherdError("E_INPUT_INVALID", `"${rung}" is not a rung`, {
+      fix: "name it as backend:model#effort or model#effort, e.g. codex:gpt-6-sol#high",
+    });
+  return canonical;
+}
+
+/** Spec §5.2 "treat like": the user maps an unscored rung onto a scored one in catalog.override.json. */
+export async function saveTreatLike(rung: string, like: string): Promise<{ rung: string; like: string }> {
+  const c = loadCatalog({ timings: false });
+  const from = canonicalOf(c, rung);
+  const to = canonicalOf(c, like);
+  if (!c.scores[to])
+    throw new CatherdError("E_CONFIG_INVALID", `${like} has no scores of its own to lend`, {
+      fix: "treat it like a scored rung; catalog_query lists them",
+    });
+  if (from === to)
+    throw new CatherdError("E_CONFIG_INVALID", `${rung} cannot be treated like itself`, {
+      fix: "name a different, scored rung",
+    });
+  if (c.scores[from])
+    throw new CatherdError(
+      "E_CONFIG_INVALID",
+      `${from} has scores of its own; a treat-like would not change it`,
+      {
+        fix: "treat-like maps only unscored rungs; catalog_query shows each rung's scores",
+      },
+    );
+  ensurePrivateDir(dirname(overridePath()));
+  await withFileLock(overridePath(), () => {
+    const cur = readOverride();
+    const next = { ...cur, schema: 1, treatLike: { ...cur.treatLike, [from]: to } };
+    const checked = OverrideSchema.safeParse(next);
+    if (!checked.success)
+      throw new CatherdError(
+        "E_CONFIG_INVALID",
+        `the treat-like ${from} → ${to} would not be a valid override`,
+        {
+          fix: checked.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`).join("; "),
+        },
+      );
+    writeJsonAtomic(overridePath(), next);
+  });
+  return { rung: from, like: to };
+}
+
+/** The backend a billing or harness key bills or isolates: `opencode-go` is a provider of the opencode backend. */
+export const backendOfKey = (key: string) => (key === "opencode-go" ? "opencode" : key);
+
+function rungRows(
+  c: Catalog,
+  backend: string,
+  model: string,
+  efforts: string[],
+  billing: Partial<Record<string, BillingMode>>,
+) {
+  return (efforts.length ? efforts : ["default"]).map((effort) => {
+    const rung = `${backend}:${model}#${effort}`;
+    const info = rungInfo(c, rung);
+    const s = scoresOf(c, info.canonical);
+    const scored: Record<string, { value: number; benchmark: string; confidence: string }> = {};
+    for (const d of DIMS) {
+      const r = s?.records[d];
+      if (r)
+        scored[d] = {
+          value: r.value,
+          benchmark: `${r.benchmark} ${r.version}`,
+          confidence: s?.via ? "inferred" : r.confidence,
+        };
+    }
+    const like = c.treatLike[info.canonical] ?? null;
+    return {
+      rung,
+      enabled: s !== null,
+      ...(s === null ? { why: "unscored: map it with catherd catalog treat-like <rung> <scored rung>" } : {}),
+      scores: scored,
+      treatLike: s?.via ? like : null,
+      cost: costOf(info.family, effort, billing[info.key] ?? DEFAULT_BILLING[info.key]),
+    };
+  });
+}
+
+export interface CatalogModel {
+  id: string;
+  name: string | null;
+  backend: string;
+  model: string;
+  billing: string;
+  efforts: string[];
+  context: number | null;
+  capabilities: Family["capabilities"] | null;
+  roles: Role[];
+  listed: boolean | null;
+  notes: Record<string, string>;
+  rungs: ReturnType<typeof rungRows>;
+}
+
+/**
+ * Spec §4.8 `catalog_query`: shipped families on each backend, plus listed models catherd cannot score;
+ * a backend listed per repository as it is listed in `f.repo` (a git toplevel), as `route` reads it.
+ */
+export function catalogQuery(
+  f: CatalogFilter,
+  billing: Partial<Record<string, BillingMode>> = {},
+): { total: number; models: CatalogModel[] } {
+  const c = loadCatalog({ timings: false, repo: f.repo });
+  const rows: CatalogModel[] = [];
+  const known = new Set<string>();
+  const entry = (backend: string, model: string, fam: Family | null): CatalogModel => {
+    const probe = rungInfo(c, `${backend}:${model}#default`);
+    return {
+      id: fam?.id ?? model,
+      name: fam?.name ?? null,
+      backend,
+      model,
+      billing: probe.key,
+      efforts: probe.efforts,
+      context: probe.context,
+      capabilities: fam?.capabilities ?? null,
+      roles: ROLES.filter((r) => capableFor(c, probe, r)),
+      listed: probe.listed,
+      notes: fam?.notes ?? {},
+      rungs: rungRows(c, backend, model, probe.efforts, billing),
+    };
+  };
+  for (const fam of c.families)
+    for (const [key, on] of Object.entries(fam.on)) {
+      const backends = key === "claude-code" ? ["claude", "claude-code"] : [backendOfKey(key)];
+      for (const b of backends) rows.push(entry(b, on.id, fam));
+      known.add(`${backendOfKey(key)}:${on.id}`);
+    }
+  for (const [backend, l] of Object.entries(c.listed))
+    for (const m of l.models) if (!known.has(`${backend}:${m.id}`)) rows.push(entry(backend, m.id, null));
+  const needle = f.text?.toLowerCase();
+  const kept = rows
+    .filter(
+      (m) =>
+        (!f.role || m.roles.includes(f.role)) &&
+        (!f.backend || m.backend === f.backend || m.billing === f.backend) &&
+        (!needle || `${m.id} ${m.model} ${m.name ?? ""}`.toLowerCase().includes(needle)) &&
+        (!f.scoredOnly || m.rungs.some((r) => r.enabled)),
+    )
+    .sort(
+      (a, b) =>
+        Number(b.rungs.some((r) => r.enabled)) - Number(a.rungs.some((r) => r.enabled)) ||
+        a.id.localeCompare(b.id) ||
+        a.backend.localeCompare(b.backend),
+    );
+  return { total: kept.length, models: kept.slice(0, f.limit) };
+}