evals-lab 0.7.0 → 0.8.0

package/lab/metrics/builtin.mjs

@@ -135,16 +135,54 @@ function verdictOf(said        )
   if (read.error || !read.value || typeof read.value !== "object") throw new Error(`the grader's reply could not be read: ${short(said)}`);
   return read.value                           ;
 }
-const ask = async (ctx           , prompt        )                  => {
+/** The grader's verdict on [prompt], its reply held to [schema] where its API
+    can hold one. A reply that still cannot be read is asked for once more;
+    a second, or a grader that failed, is the grader's error -- said as one,
+    so it is not read as the reply failing. [ok] says whether a verdict read
+    is one this metric can use. */
+async function graded(ctx           , prompt        , schema                         ,
+                      ok                                               = () => true)                                        {
   if (!ctx.ask) throw new Error("a model-graded metric needs a grader, and this run has none");
-  return ctx.ask(prompt);
-};
+  let last = "";
+  for (let attempt = 0; attempt < 2; attempt++) {
+    let said        ;
+    try { said = await ctx.ask(prompt, schema); } catch (e) { throw new Error(`grader error: ${e instanceof Error ? e.message : String(e)}`); }
+    try {
+      const v = verdictOf(said);
+      if (ok(v)) return v;
+      last = `the grader said ${short(JSON.stringify(v))}`;
+    } catch (e) { last = e instanceof Error ? e.message : String(e); }
+  }
+  throw new Error(`grader error: ${last}`);
+}
 const gradedPass = (v                              , threshold        )                => {
   const score = typeof v.score === "number" ? Math.max(0, Math.min(1, v.score)) : v.pass === true ? 1 : 0;
   const pass = typeof v.pass === "boolean" ? v.pass && score >= threshold : score >= threshold;
   return { pass, score, reason: str(v.reason) || (pass ? "the grader passed it" : "the grader failed it") };
 };
+/** The shape a pass-or-fail grader answers in, for its API to hold it to. */
+const VERDICT_SCHEMA = { type: "object", additionalProperties: false, required: ["pass", "score", "reason"],
+  properties: { pass: { type: "boolean" }, score: { type: "number" }, reason: { type: "string" } } };
 const REPLY = 'Reply with one JSON object and nothing else: {"pass": true or false, "score": a number from 0 to 1, "reason": "one sentence"}.';
+/** What the grader is told of the task: what the target was asked, and the
+    guidance the metric adds, where there is either. */
+const taskOf = (input             , guidance         )         =>
+  (input.asked?.trim() ? `The task, as it was asked:\n${input.asked.trim()}\n\n` : "")
+  + (str(guidance).trim() ? `Guidance:\n${str(guidance).trim()}\n\n` : "");
+
+/** A grader's three verdicts on a reply set beside the recorded one; any of
+    them may be what a metric expects. */
+const JUDGEMENTS = [{ value: "better", label: "Better" }, { value: "same", label: "Same" }, { value: "worse", label: "Worse" }];
+const JUDGEMENT_SCORE                         = { better: 1, same: 0.5, worse: 0 };
+/** What a judgement metric expects: the verdicts set, or -- set before
+    Expected was offered -- the one rule there was, worse fails. */
+const expectedOf = (m        )           =>
+  (Array.isArray(m.expected) ? m.expected.map(String) : ["better", "same"]);
+/** The expected verdicts as a reader says them: "better or same". */
+const said = (values                   )         => {
+  const words = values.map((v) => v.toLowerCase());
+  return words.length < 2 ? words.join("") : `${words.slice(0, -1).join(", ")} or ${words.at(-1)}`;
+};
 
 // The families the Add metric picker lists the metrics under: what each reads.
 const TEXT = "The reply's text", RESULT = "The result", RECORDED = "Recorded reply", GRADED = "Model-graded";
@@ -342,6 +380,7 @@ const metrics                              = {
   // reply compared with the reply it is says nothing (docs/workflow-sources.md).
   "same-as-recorded": {
     family: RECORDED,
+    parsedOnly: true,
     label: "Same as recorded",
     description: "Passes when the reply is what production recorded for the same item.",
     options: [],
@@ -356,6 +395,7 @@ const metrics                              = {
   },
   "fields-equal-recorded": {
     family: RECORDED,
+    parsedOnly: true,
     label: "Fields as recorded",
     description: "Passes when the fields listed are what the recorded reply had.",
     options: [{ key: "fields", label: "Fields", type: "textarea" }],
@@ -374,6 +414,7 @@ const metrics                              = {
   },
   "same-parse-as-recorded": {
     family: RECORDED,
+    parsedOnly: true,
     label: "Parses as recorded",
     description: "Passes when the reply parses as JSON exactly when the recorded reply did.",
     options: [],
@@ -390,43 +431,58 @@ const metrics                              = {
     label: "Rubric",
     description: "Asks the grader whether the reply meets the rubric.",
     graded: true,
-    options: [{ key: "rubric", label: "Rubric", type: "textarea" }, { key: "threshold", label: "Pass at", type: "number" }],
+    parsedOnly: true,
+    options: [{ key: "rubric", label: "Rubric", type: "textarea" }, { key: "threshold", label: "Expected: score at least", type: "number" }],
     defaults: () => ({ rubric: "", threshold: 0.5 }),
     validate: (m, at, bad) => needs(m, ["rubric"], at, bad),
-    score: async (input, m, ctx) => gradedPass(verdictOf(await ask(ctx,
-      `You are grading an output against a rubric.\n\nRubric:\n${str(m.rubric)}\n\nOutput:\n${input.text}\n\n${REPLY}`)),
-      num(m.threshold) ?? 0.5),
+    score: async (input, m, ctx) => gradedPass(await graded(ctx,
+      `You are grading an output against a rubric.\n\n${taskOf(input, "")}Rubric:\n${str(m.rubric)}\n\nOutput:\n${input.text}\n\n${REPLY}`,
+      VERDICT_SCHEMA), num(m.threshold) ?? 0.5),
   },
   factuality: {
     family: GRADED,
     label: "Factual",
     description: "Asks the grader whether the reply agrees with the reference, or the recorded reply.",
     graded: true,
-    options: [{ key: "reference", label: "Reference", type: "textarea" }, { key: "threshold", label: "Pass at", type: "number" }],
+    parsedOnly: true,
+    options: [{ key: "reference", label: "Reference", type: "textarea" }, { key: "threshold", label: "Expected: score at least", type: "number" }],
     defaults: () => ({ reference: "", threshold: 0.5 }),
     // A blank reference is the recorded reply.
-    score: async (input, m, ctx) => gradedPass(verdictOf(await ask(ctx,
+    score: async (input, m, ctx) => gradedPass(await graded(ctx,
       `You are checking an output for factual consistency with a reference. Differences in wording or detail are `
-      + `fine; a contradiction, or a fact the reference does not support, is not.\n\nReference:\n`
-      + `${str(m.reference).trim() || recorded(input)}\n\nOutput:\n${input.text}\n\n${REPLY}`)), num(m.threshold) ?? 0.5),
+      + `fine; a contradiction, or a fact the reference does not support, is not.\n\n${taskOf(input, "")}Reference:\n`
+      + `${str(m.reference).trim() || recorded(input)}\n\nOutput:\n${input.text}\n\n${REPLY}`, VERDICT_SCHEMA), num(m.threshold) ?? 0.5),
   },
   "judge-vs-production": {
     family: GRADED,
     label: "Judged against recorded",
     description: "Asks the grader whether the reply is better than, the same as or worse than the recorded one.",
     graded: true,
-    options: [{ key: "task", label: "Task", type: "textarea" }],
-    defaults: () => ({ task: "" }),
+    parsedOnly: true,
+    expects: true,
+    options: [{ key: "expected", label: "Expected", type: "multi", choices: JUDGEMENTS },
+              { key: "task", label: "Additional guidance (optional)", type: "textarea" }],
+    defaults: () => ({ expected: ["better", "same"], task: "" }),
+    validate: (m, at, bad) => {
+      const known = JUDGEMENTS.map((j) => j.value);
+      if (m.expected !== undefined && (!Array.isArray(m.expected) || !m.expected.length
+          || m.expected.some((v) => !known.includes(String(v))))) {
+        bad.push(`${at}: Expected needs at least one of ${said(known)}`);
+      }
+    },
     score: async (input, m, ctx) => {
-      const v = verdictOf(await ask(ctx,
+      if (ctx.recordedTarget) return null;
+      const v = await graded(ctx,
         `Two outputs answer the same task. Say whether the NEW one is better than, the same as, or worse than the `
-        + `RECORDED one.${str(m.task).trim() ? `\n\nTask:\n${str(m.task)}` : ""}\n\nRECORDED:\n${recorded(input)}\n\n`
+        + `RECORDED one.\n\n${taskOf(input, m.task)}RECORDED:\n${recorded(input)}\n\n`
         + `NEW:\n${input.text}\n\nReply with one JSON object and nothing else: {"verdict": "better" or "same" or "worse", `
-        + `"reason": "one sentence"}.`));
-      const verdict = str(v.verdict).toLowerCase();
-      if (!["better", "same", "worse"].includes(verdict)) throw new Error(`the grader said ${short(JSON.stringify(v))}`);
-      return { pass: verdict !== "worse", score: verdict === "better" ? 1 : verdict === "same" ? 0.5 : 0,
-               reason: `${verdict}: ${str(v.reason)}` };
+        + `"reason": "one sentence"}.`,
+        { type: "object", additionalProperties: false, required: ["verdict", "reason"],
+          properties: { verdict: { type: "string", enum: JUDGEMENTS.map((j) => j.value) }, reason: { type: "string" } } },
+        (r) => str(r.verdict).toLowerCase() in JUDGEMENT_SCORE);
+      const verdict = str(v.verdict).toLowerCase(), expected = expectedOf(m);
+      return { pass: expected.includes(verdict), score: JUDGEMENT_SCORE[verdict] ,
+               reason: `${verdict} (expected ${said(expected)}): ${str(v.reason)}` };
     },
   },
 };

package/lab/run-evals.js

@@ -605,8 +605,12 @@ function graderFor(run) {
       if (!c) return undefined;
       if (!made.has(ref.id)) {
         const link = reach(c, core.keyVar(ref.id, c.slug));
-        made.set(ref.id, async prompt => {
-          const r = await ask(link, { id: ref.id, ...c }, prompt, null);
+        made.set(ref.id, async (prompt, schema) => {
+          // Held to the metric's schema where the grader's API can hold a
+          // reply to one; a model or server that refuses the schema is asked
+          // again without it, the prompt alone saying the shape.
+          let r = await ask(link, { id: ref.id, ...c }, prompt, null, schema);
+          if (r.error && schema) r = await ask(link, { id: ref.id, ...c }, prompt, null);
           if (r.error) throw new Error(`the grader ${c.name || ref.id} failed: ${r.error}`);
           return r.raw ?? "";
         });
@@ -660,13 +664,13 @@ async function send(to, c, step, cell, record) {
   }
 }
 
-async function ask(to, c, prompt, dataUrl) {
+async function ask(to, c, prompt, dataUrl, schema = null) {
   const t0 = Date.now();
   const said = more => ({ ms: Date.now() - t0, conn: to.name, ...more });
   try {
     // The connection's type builds the whole request: the URL, the headers
     // (Anthropic's key travels in x-api-key), and the body.
-    const req = core.connectionRequest(c, prompt, dataUrl, to.base, to.key);
+    const req = core.connectionRequest(c, prompt, dataUrl, to.base, to.key, schema);
     const r = await fetch(req.url, {
       method: "POST",
       headers: { "Content-Type": "application/json", ...req.headers },