catherd-cli 0.2.1 → 1.0.0

package/src/domain/jev.ts

@@ -0,0 +1,283 @@
+import { z } from "zod";
+import { type Difficulty, KINDS, type Kind, parseLaneHeader } from "./lane.ts";
+import { scrubKeyShapes, scrubSecretAssignments } from "./secrets.ts";
+
+// Spec §5.5 and research 2026-09-25-jev.md: Jev's three question types, catherd's question sets as
+// versioned data (catalog/jev.json), the trimmed lane state, and the decision rules.
+
+const ChoiceQ = z.object({
+  type: z.literal("choice"),
+  instructions: z.string(),
+  criteria: z.record(z.string(), z.string()),
+});
+const ScoreQ = z.object({
+  type: z.literal("score"),
+  instructions: z.string(),
+  criteria: z.array(z.string()).min(2).max(10),
+});
+const NoulQ = z.object({
+  type: z.literal("noul"),
+  instructions: z.string(),
+  criteria: z.object({ true: z.string(), false: z.string() }).optional(),
+});
+const QuestionSchema = z.discriminatedUnion("type", [ChoiceQ, ScoreQ, NoulQ]);
+export type JevQuestion = z.infer<typeof QuestionSchema>;
+
+const RouteRuleSchema = z.object({
+  kindMin: z.number().min(0).max(1),
+  trackMin: z.number().gt(0.5).max(1),
+  /** exactly two levels each, in order: copy then build, logic then hard */
+  trackA: z.array(z.string()).length(2),
+  trackB: z.array(z.string()).length(2),
+});
+export type RouteRule = z.infer<typeof RouteRuleSchema>;
+const VerdictRuleSchema = z.object({ min: z.number().min(0).max(1), fallback: z.string() });
+export type VerdictRule = z.infer<typeof VerdictRuleSchema>;
+
+export const JevFileSchema = z.looseObject({
+  schema: z.literal(1),
+  model: z.string(),
+  sets: z.object({
+    "route-v2": z.object({ questions: z.record(z.string(), QuestionSchema), rule: RouteRuleSchema }),
+    finding: z.object({ questions: z.record(z.string(), QuestionSchema), rule: VerdictRuleSchema }),
+    "same-defect": z.object({ questions: z.record(z.string(), QuestionSchema), rule: VerdictRuleSchema }),
+  }),
+});
+export type JevFile = z.infer<typeof JevFileSchema>;
+export type SetName = keyof JevFile["sets"];
+
+/** The answers each verdict set's caller accepts; a set's `fallback` must be one of them. */
+export const VERDICT_OPTIONS = {
+  finding: ["design", "code", "unclear"],
+  "same-defect": ["yes", "no"],
+} as const;
+
+/** JSON with object keys sorted at every depth, so equal data always hashes the same. */
+export function canonicalJson(v: unknown): string {
+  if (Array.isArray(v)) return `[${v.map(canonicalJson).join(",")}]`;
+  if (v !== null && typeof v === "object")
+    return `{${Object.keys(v)
+      .filter((k) => (v as Record<string, unknown>)[k] !== undefined)
+      .sort()
+      .map((k) => `${JSON.stringify(k)}:${canonicalJson((v as Record<string, unknown>)[k])}`)
+      .join(",")}}`;
+  return JSON.stringify(v) ?? "null";
+}
+
+export const sha256 = (s: string): string => new Bun.CryptoHasher("sha256").update(s).digest("hex");
+
+/** `<set>#<8 hex>` over the model, questions and rule: any change to one names a new question set. */
+export function questionSetId(f: JevFile, name: SetName): string {
+  const set = f.sets[name];
+  return `${name}#${sha256(canonicalJson({ model: f.model, questions: set.questions, rule: set.rule })).slice(0, 8)}`;
+}
+
+/** The per-run cache key of one request. */
+export const requestKey = (model: string, state: unknown, questions: unknown): string =>
+  sha256(canonicalJson({ model, state, questions }));
+
+/**
+ * Key-shaped strings (domain/secrets.ts) and `key = value` secrets replaced by [secret]; the pair, a
+ * bearer and a URL's user keep their names.
+ */
+export const scrubSecrets = (text: string): string =>
+  scrubSecretAssignments(scrubKeyShapes(text, "[secret]"), "[secret]");
+
+/** Research §5.3: code is not what Jev judges, and long state rots its answers. */
+export const BODY_MAX = 6000;
+const HEADER = /^\s*[*_]*(owns|fast check|kind|difficulty)[*_]*\s*:/i;
+
+export interface LaneState {
+  title: string | null;
+  owns: string[];
+  fast_check: string | null;
+  body: string;
+}
+
+/**
+ * Spec §5.5: `{ title, owns, fast_check, body }`. The body drops the header lines (so Jev never echoes
+ * the architect's own Kind/Difficulty), fenced code, and secrets, and is capped at BODY_MAX characters.
+ * Every field is scrubbed of secrets.
+ */
+export function laneState(text: string): LaneState {
+  const h = parseLaneHeader(text);
+  const body = dropFences(text)
+    .split("\n")
+    .filter((l) => !/^#\s/.test(l) && !HEADER.test(l))
+    .join("\n")
+    .trim();
+  return {
+    title: h.title === null ? null : scrubSecrets(h.title),
+    owns: h.owns.map(scrubSecrets).filter(Boolean),
+    fast_check: h.fastCheck === null ? null : scrubSecrets(h.fastCheck),
+    body: capBody(scrubSecrets(body)),
+  };
+}
+
+/** Fenced code, indented or not, becomes "[code omitted]"; a fence left open runs to the end of the text, as Markdown renders it. */
+const dropFences = (text: string): string =>
+  text.replace(/^[ \t]*(`{3,}|~{3,})[^\n]*(?:\n[\s\S]*?^[ \t]*\1[^\n]*$|[\s\S]*$)/gm, "[code omitted]");
+
+/** Capped at BODY_MAX characters, never ending the cut on half a surrogate pair. */
+const capBody = (text: string): string =>
+  text.length > BODY_MAX ? `${text.slice(0, BODY_MAX).replace(/[\uD800-\uDBFF]$/, "")}…` : text;
+
+/** Free text sent to Jev (a finding, a defect): no fenced code, no secrets, capped like a lane's body. */
+export function scrubFree(text: string): string {
+  return capBody(scrubSecrets(dropFences(text).trim()));
+}
+
+const Prob = z.record(z.string(), z.number().min(0).max(1));
+const AnswerSchema = z.discriminatedUnion("type", [
+  z.object({ type: z.literal("choice"), choice: z.string(), probabilities: Prob, confidence: z.number() }),
+  z.object({
+    type: z.literal("score"),
+    score: z.number(),
+    probabilities: Prob,
+    confidence: z.number(),
+  }),
+  z.object({ type: z.literal("noul"), noul: z.number().min(0).max(1) }),
+]);
+type JevAnswer = z.infer<typeof AnswerSchema>;
+export type JevAnswers = Record<string, JevAnswer>;
+
+const ReplySchema = z.object({
+  model: z.string(),
+  answers: z.record(z.string(), z.unknown()),
+  usage: z.object({ input_tokens: z.number(), output_tokens: z.number() }).optional(),
+});
+
+export type ParsedReply =
+  | {
+      ok: true;
+      model: string;
+      usage: { input_tokens: number; output_tokens: number } | null;
+      answers: JevAnswers;
+    }
+  | { ok: false; error: string };
+
+/** Every question answered, each in its own type and within its options, or an error naming the first. */
+export function parseReply(questions: Record<string, JevQuestion>, body: unknown): ParsedReply {
+  const r = ReplySchema.safeParse(body);
+  if (!r.success) return { ok: false, error: "unexpected response" };
+  const answers: JevAnswers = {};
+  for (const [id, q] of Object.entries(questions)) {
+    const a = AnswerSchema.safeParse(r.data.answers[id]);
+    if (!a.success || a.data.type !== q.type) return { ok: false, error: `no valid answer to ${id}` };
+    if (a.data.type === "choice" && q.type === "choice" && !Object.hasOwn(q.criteria, a.data.choice))
+      return { ok: false, error: `no valid answer to ${id}` };
+    if (
+      a.data.type === "score" &&
+      q.type === "score" &&
+      Object.keys(a.data.probabilities).some((l) => !/^\d+$/.test(l) || Number(l) >= q.criteria.length)
+    )
+      return { ok: false, error: `no valid answer to ${id}` };
+    answers[id] = a.data;
+  }
+  return { ok: true, model: r.data.model, usage: r.data.usage ?? null, answers };
+}
+
+const pOf = (a: JevAnswer | undefined, option: string): number =>
+  a && a.type !== "noul" ? (a.probabilities[option] ?? 0) : 0;
+
+export interface RouteJudgement {
+  /** the kind, when p_max ≥ kindMin and it is one of catherd's kinds */
+  kind: Kind | null;
+  pKind: number | null;
+  /** "A" (copy/build) or "B" (logic/hard) on summed probabilities, null in the dead band */
+  track: "A" | "B" | null;
+  difficulty: Difficulty | null;
+  pA: number | null;
+  pB: number | null;
+  /** the atomic questions, logged for calibration only */
+  nouls: Record<string, number>;
+  rule: string;
+}
+
+/**
+ * Spec §5.5's decision rule. Confidence is `(n·p_max − 1)/(n − 1)`, so it means different things for
+ * different option counts; the rule reads probabilities instead: the kind at p_max ≥ kindMin, the track
+ * when the summed probability of its levels reaches trackMin, and neither in between (the dead band).
+ */
+export function judgeRoute(rule: RouteRule, answers: JevAnswers): RouteJudgement {
+  const k = answers.kind;
+  const d = answers.difficulty;
+  const nouls: Record<string, number> = {};
+  for (const [id, a] of Object.entries(answers)) if (a.type === "noul") nouls[id] = a.noul;
+  let kind: Kind | null = null;
+  let pKind: number | null = null;
+  if (k && k.type === "choice") {
+    const [top, p] = Object.entries(k.probabilities).reduce((m, e) => (e[1] > m[1] ? e : m), ["", -1]);
+    pKind = p < 0 ? null : p;
+    if (p >= rule.kindMin && (KINDS as readonly string[]).includes(top)) kind = top as Kind;
+  }
+  if (!d || d.type !== "score")
+    return { kind, pKind, track: null, difficulty: null, pA: null, pB: null, nouls, rule: "no difficulty" };
+  // Rounded before comparing: 0.1 + 0.7 is 0.7999999999999999 in floating point, and must reach 0.8.
+  const round = (x: number) => Math.round(x * 1e6) / 1e6;
+  const sum = (levels: string[]) => round(levels.reduce((s, l) => s + pOf(d, l), 0));
+  const pA = sum(rule.trackA);
+  const pB = sum(rule.trackB);
+  if (pA >= rule.trackMin)
+    return {
+      kind,
+      pKind,
+      track: "A",
+      difficulty: pOf(d, rule.trackA[0] ?? "") > pOf(d, rule.trackA[1] ?? "") ? "copy" : "build",
+      pA,
+      pB,
+      nouls,
+      rule: `P(A) ${pA} ≥ ${rule.trackMin}`,
+    };
+  if (pB >= rule.trackMin)
+    return {
+      kind,
+      pKind,
+      track: "B",
+      difficulty: pOf(d, rule.trackB[1] ?? "") > pOf(d, rule.trackB[0] ?? "") ? "hard" : "logic",
+      pA,
+      pB,
+      nouls,
+      rule: `P(B) ${pB} ≥ ${rule.trackMin}`,
+    };
+  return {
+    kind,
+    pKind,
+    track: null,
+    difficulty: null,
+    pA,
+    pB,
+    nouls,
+    rule: `P(A) ${pA}, P(B) ${pB}: both below ${rule.trackMin}`,
+  };
+}
+
+export interface Verdict<T extends string> {
+  value: T;
+  /** the chosen answer's probability, when Jev answered */
+  probability: number | null;
+  /** Jev's own confidence statistic, when it answered */
+  confidence: number | null;
+  source: "jev" | "default";
+}
+
+/** Spec §5.5: `finding` and `same-defect` keep their 0.x meaning, with thresholds on probability. */
+export function judgeVerdict<T extends string>(
+  rule: VerdictRule,
+  id: string,
+  options: readonly T[],
+  answers: JevAnswers | null,
+): Verdict<T> {
+  const a = answers?.[id];
+  const fallback = rule.fallback as T;
+  if (!a || a.type !== "choice" || !(options as readonly string[]).includes(a.choice))
+    return { value: fallback, probability: null, confidence: null, source: "default" };
+  const p = a.probabilities[a.choice] ?? 0;
+  const sure = p >= rule.min;
+  return {
+    value: sure ? (a.choice as T) : fallback,
+    probability: p,
+    confidence: a.confidence,
+    source: sure ? "jev" : "default",
+  };
+}

package/src/domain/lane.ts

@@ -0,0 +1,69 @@
+import { CatherdError } from "./errors.ts";
+
+export const KINDS = ["repo_code", "terminal", "ui", "prose", "research"] as const;
+export type Kind = (typeof KINDS)[number];
+export const DIFFICULTIES = ["copy", "build", "logic", "hard"] as const;
+export type Difficulty = (typeof DIFFICULTIES)[number];
+
+export interface LaneHeader {
+  title: string | null;
+  owns: string[];
+  fastCheck: string | null;
+  kind: Kind | null;
+  difficulty: Difficulty | null;
+}
+
+const unquote = (s: string) =>
+  s
+    .trim()
+    .replace(/^`+|`+$/g, "")
+    .trim();
+
+/** The value of the first `Label:` line, allowing `**Label:**` and `_Label_:` emphasis. */
+function field(text: string, label: string): string | null {
+  const re = new RegExp(`^\\s*[*_]*${label}[*_]*\\s*:[*_]*\\s*(.*)$`, "i");
+  for (const line of text.split("\n")) {
+    const m = re.exec(line);
+    if (m) return (m[1] ?? "").trim();
+  }
+  return null;
+}
+
+const oneOf = <T extends string>(v: string | null, options: readonly T[]): T | null =>
+  v !== null && (options as readonly string[]).includes(unquote(v)) ? (unquote(v) as T) : null;
+
+/** Canonical form: no "." or empty segments, a trailing "/" kept as a directory marker. */
+export function normalizeOwned(p: string): string {
+  const s = unquote(p);
+  const parts = s.split("/").filter((seg) => seg !== "" && seg !== ".");
+  if (s.startsWith("/") || parts.length === 0 || parts.includes(".."))
+    throw new CatherdError(
+      "E_LANE_INVALID",
+      `owned path "${p}" must be relative to the repo and stay inside it`,
+      {
+        fix: "write Owns: paths like src/foo/ or src/bar.ts",
+      },
+    );
+  return parts.join("/") + (s.endsWith("/") ? "/" : "");
+}
+
+export function parseLaneHeader(text: string): LaneHeader {
+  const title = /^#\s+(.+)$/m.exec(text)?.[1]?.trim() ?? null;
+  const owns = (field(text, "owns") ?? "").split(",").map(unquote).filter(Boolean).map(normalizeOwned);
+  const check = field(text, "fast check");
+  return {
+    title,
+    owns,
+    fastCheck: check ? unquote(check) || null : null,
+    kind: oneOf(field(text, "kind"), KINDS),
+    difficulty: oneOf(field(text, "difficulty"), DIFFICULTIES),
+  };
+}
+
+const bare = (p: string) => p.replace(/\/+$/, "");
+const covers = (outer: string, inner: string) => inner === outer || inner.startsWith(`${outer}/`);
+
+/** The entries of `a` that overlap any entry of `b`; a path covers everything under it. */
+export function overlaps(a: string[], b: string[]): string[] {
+  return a.filter((x) => b.some((y) => covers(bare(x), bare(y)) || covers(bare(y), bare(x))));
+}

package/src/domain/profile-rules.ts

@@ -0,0 +1,205 @@
+import {
+  billingKeyOf,
+  type Catalog,
+  capableFor,
+  effortOffered,
+  ROLE_NEEDS,
+  type RungInfo,
+  rungInfo,
+  scoresOf,
+} from "./catalog.ts";
+import { type Rung, tryParseRung } from "./ids.ts";
+import { DIFFICULTIES, KINDS } from "./lane.ts";
+import { type Profile, type ProfileDoc, unknownValues } from "./profile.ts";
+import { DEFAULT_ACCESS, ROLES, type Role } from "./roles.ts";
+import { candidates, clearsBar, type RoutingProfile } from "./select.ts";
+
+/** One finding of `validate`: where in the profile, what is wrong, and the action that fixes it. */
+export interface Issue {
+  path: string;
+  message: string;
+  fix?: string;
+}
+
+export interface Validation {
+  errors: Issue[];
+  warnings: Issue[];
+}
+
+const TREAT_LIKE_FIX = (rung: string) =>
+  `map it with: catherd catalog treat-like ${rung} <a scored rung>; catalog_query lists them`;
+
+export const routingProfileOf = (p: Profile, role: Role): RoutingProfile => ({
+  objective: p.objective,
+  billing: p.billing,
+  role: p.roles[role],
+});
+
+/**
+ * Spec §7.1 and §4.5: a stand-in on the same quota would be out of quota too. The billing key names the
+ * quota, except that the native `claude` path and headless `claude-code` both draw on the Claude plan.
+ */
+export const quotaOf = (r: Rung): string => (billingKeyOf(r) === "claude" ? "claude-code" : billingKeyOf(r));
+
+/** Whether a rung's scores are catherd's guess: borrowed through a treat-like, or only `inferred` ones. */
+export function inferredScores(c: Catalog, info: RungInfo): { inferred: boolean; via: string | null } {
+  const s = scoresOf(c, info.canonical);
+  if (!s) return { inferred: false, via: null };
+  const records = Object.values(s.records);
+  return { inferred: s.via !== null || records.every((r) => r.confidence === "inferred"), via: s.via };
+}
+
+/**
+ * Spec §7.1. Errors: the worker disabled; an enabled role with no usable rung; an unscored rung without a
+ * treat-like; a failover stand-in unscored or on the same quota; a rung whose backend catherd cannot run
+ * (`backends` lists those it can, `claude` included). Everything else worth knowing is a warning: an access
+ * mode other than the role's default, an effort or model the last listing does not offer, a stand-in
+ * that never runs. With the stored `doc` `p` came from, a value this catherd does not know is also a
+ * warning, one that says how the value is read.
+ */
+export function validateProfile(
+  p: Profile,
+  c: Catalog,
+  backends: readonly string[],
+  doc?: ProfileDoc,
+): Validation {
+  const errors: Issue[] = [];
+  const warnings: Issue[] = [];
+  for (const u of doc ? unknownValues(doc) : [])
+    warnings.push({
+      path: u.path,
+      message: `"${u.value}" is not a value this catherd knows (a newer one wrote it?); it is read as ${u.readAs}`,
+      fix: "upgrade catherd (bun add -g catherd-cli@latest), or set a value this version knows",
+    });
+  if (!p.roles.worker.enabled)
+    errors.push({
+      path: "roles.worker.enabled",
+      message: "the worker cannot be disabled",
+      fix: "catherd profile set roles.worker.enabled true",
+    });
+
+  for (const role of ROLES) {
+    const rc = p.roles[role];
+    const at = `roles.${role}`;
+    if (rc.access !== DEFAULT_ACCESS[role])
+      warnings.push({
+        path: `${at}.access`,
+        message: `${role} runs ${rc.access}; catherd's default for it is ${DEFAULT_ACCESS[role]}`,
+      });
+    if (!rc.enabled) continue;
+    for (const rung of rc.rungs) {
+      const r = tryParseRung(rung);
+      if (!r) {
+        errors.push({
+          path: `${at}.rungs`,
+          message: `"${rung}" is not a rung`,
+          fix: "write it as <backend>:<model>#<effort>",
+        });
+        continue;
+      }
+      if (!backends.includes(r.backend)) {
+        errors.push({
+          path: `${at}.rungs`,
+          message: `${rung}: catherd cannot run ${r.backend} yet`,
+          fix: `use a rung on ${backends.join(", ")}`,
+        });
+        continue;
+      }
+      const info = rungInfo(c, rung);
+      if (!capableFor(c, info, role))
+        errors.push({
+          path: `${at}.rungs`,
+          message: `${rung} cannot fill the ${role} role (it needs ${Object.keys(ROLE_NEEDS[role]).join(" + ")})`,
+          fix: `catalog_query({ role: "${role}" }) lists the models that can`,
+        });
+      if (info.efforts.length > 0 && !effortOffered(info))
+        errors.push({
+          path: `${at}.rungs`,
+          message: `${r.model} has no effort "${r.effort}" on ${r.backend} (it has ${info.efforts.join(", ")})`,
+        });
+      else if (info.efforts.length === 0 && r.effort !== "default")
+        warnings.push({
+          path: `${at}.rungs`,
+          message: `${rung}: catherd cannot check its effort until ${r.backend} lists its models`,
+          fix: "catherd catalog refresh",
+        });
+      if (info.listed === false)
+        warnings.push({
+          path: `${at}.rungs`,
+          message: `${r.backend}'s last listing does not offer ${r.model}; routing skips it`,
+        });
+      if (!scoresOf(c, info.canonical))
+        errors.push({ path: `${at}.rungs`, message: `${rung} is unscored`, fix: TREAT_LIKE_FIX(rung) });
+    }
+    if (rc.defaultRung !== undefined && !rc.rungs.includes(rc.defaultRung))
+      errors.push({
+        path: `${at}.defaultRung`,
+        message: `${rc.defaultRung} is not one of the ${role}'s rungs`,
+        fix: `catherd profile set ${at}.defaultRung null, or add it to ${at}.rungs`,
+      });
+    const usable = candidates(c, routingProfileOf(p, role), role);
+    if (usable.length === 0)
+      errors.push({
+        path: `${at}.rungs`,
+        message: `the ${role} role has no usable rung`,
+        fix: `enable a scored rung its backend offers, or turn the role off with: catherd profile set ${at}.enabled false`,
+      });
+    if (usable.length > 1)
+      // Claude rungs have no honesty score, so they clear no shipped bar: select() starts on one only as the
+      // default rung (when nothing clears) and never climbs onto it
+      for (const x of usable)
+        if (
+          quotaOf(x.info.parsed) === "claude-code" &&
+          !KINDS.some((k) => DIFFICULTIES.some((d) => clearsBar(c, x, k, d)))
+        )
+          warnings.push({
+            path: `${at}.rungs`,
+            message: `${x.rung} clears no routing bar, so a lane starts on it only as the role's default rung and never climbs onto it`,
+          });
+    if (role === "worker" && usable.length > 1) {
+      const missed = KINDS.flatMap((k) =>
+        DIFFICULTIES.filter((d) => !usable.some((x) => clearsBar(c, x, k, d))).map((d) => `${k}/${d}`),
+      );
+      if (missed.length)
+        warnings.push({
+          path: `${at}.rungs`,
+          message: `no worker rung clears the bar for ${missed.join(", ")}; those lanes start at the default rung`,
+        });
+    }
+  }
+
+  const ladders = new Set(ROLES.filter((r) => p.roles[r].enabled).flatMap((r) => p.roles[r].rungs));
+  for (const [from, to] of Object.entries(p.failover)) {
+    const at = `failover.${from}`;
+    const a = tryParseRung(from);
+    const b = tryParseRung(to);
+    if (!a || !b) {
+      errors.push({
+        path: at,
+        message: `"${!a ? from : to}" is not a rung`,
+        fix: "write it as <backend>:<model>#<effort>",
+      });
+      continue;
+    }
+    if (!backends.includes(b.backend)) {
+      errors.push({ path: at, message: `stand-in ${to}: catherd cannot run ${b.backend} yet` });
+      continue;
+    }
+    if (!scoresOf(c, rungInfo(c, to).canonical))
+      errors.push({ path: at, message: `stand-in ${to} is unscored`, fix: TREAT_LIKE_FIX(to) });
+    if (quotaOf(a) === quotaOf(b))
+      errors.push({
+        path: at,
+        message: `stand-in ${to} draws on the same quota as ${from}, which is out when ${from} hits its limit`,
+        fix: "name a stand-in on another backend or plan",
+      });
+    if (b.backend === "claude")
+      warnings.push({
+        path: at,
+        message: `stand-in ${to} is a native subagent: the orchestrator must start it, dispatch cannot`,
+      });
+    if (!ladders.has(from))
+      warnings.push({ path: at, message: `${from} is on no enabled role's ladder, so this never runs` });
+  }
+  return { errors, warnings };
+}