evals-lab 0.6.0 → 0.8.0

package/lab/metrics/builtin.mjs

@@ -120,10 +120,10 @@ function holds(input             , m        , value        , except
 /** A requirement as Results names it: a term, or the group it was. */
 const spoken = (g          )              => (g.length === 1 ? g[0]  : g);
 
-/** Production's reply, or the reason a metric that compares with it cannot. */
-const production = (input             )         => {
-  if (input.production == null) throw new Error("this item has no production reply to compare with");
-  return input.production;
+/** The recorded reply, or the reason a metric that compares with it cannot. */
+const recorded = (input             )         => {
+  if (input.recorded == null) throw new Error("this item has no recorded reply to compare with");
+  return input.recorded;
 };
 
 // ---- model-graded -----------------------------------------------------------
@@ -135,19 +135,57 @@ function verdictOf(said        )
   if (read.error || !read.value || typeof read.value !== "object") throw new Error(`the grader's reply could not be read: ${short(said)}`);
   return read.value                           ;
 }
-const ask = async (ctx           , prompt        )                  => {
+/** The grader's verdict on [prompt], its reply held to [schema] where its API
+    can hold one. A reply that still cannot be read is asked for once more;
+    a second, or a grader that failed, is the grader's error -- said as one,
+    so it is not read as the reply failing. [ok] says whether a verdict read
+    is one this metric can use. */
+async function graded(ctx           , prompt        , schema                         ,
+                      ok                                               = () => true)                                        {
   if (!ctx.ask) throw new Error("a model-graded metric needs a grader, and this run has none");
-  return ctx.ask(prompt);
-};
+  let last = "";
+  for (let attempt = 0; attempt < 2; attempt++) {
+    let said        ;
+    try { said = await ctx.ask(prompt, schema); } catch (e) { throw new Error(`grader error: ${e instanceof Error ? e.message : String(e)}`); }
+    try {
+      const v = verdictOf(said);
+      if (ok(v)) return v;
+      last = `the grader said ${short(JSON.stringify(v))}`;
+    } catch (e) { last = e instanceof Error ? e.message : String(e); }
+  }
+  throw new Error(`grader error: ${last}`);
+}
 const gradedPass = (v                              , threshold        )                => {
   const score = typeof v.score === "number" ? Math.max(0, Math.min(1, v.score)) : v.pass === true ? 1 : 0;
   const pass = typeof v.pass === "boolean" ? v.pass && score >= threshold : score >= threshold;
   return { pass, score, reason: str(v.reason) || (pass ? "the grader passed it" : "the grader failed it") };
 };
+/** The shape a pass-or-fail grader answers in, for its API to hold it to. */
+const VERDICT_SCHEMA = { type: "object", additionalProperties: false, required: ["pass", "score", "reason"],
+  properties: { pass: { type: "boolean" }, score: { type: "number" }, reason: { type: "string" } } };
 const REPLY = 'Reply with one JSON object and nothing else: {"pass": true or false, "score": a number from 0 to 1, "reason": "one sentence"}.';
+/** What the grader is told of the task: what the target was asked, and the
+    guidance the metric adds, where there is either. */
+const taskOf = (input             , guidance         )         =>
+  (input.asked?.trim() ? `The task, as it was asked:\n${input.asked.trim()}\n\n` : "")
+  + (str(guidance).trim() ? `Guidance:\n${str(guidance).trim()}\n\n` : "");
+
+/** A grader's three verdicts on a reply set beside the recorded one; any of
+    them may be what a metric expects. */
+const JUDGEMENTS = [{ value: "better", label: "Better" }, { value: "same", label: "Same" }, { value: "worse", label: "Worse" }];
+const JUDGEMENT_SCORE                         = { better: 1, same: 0.5, worse: 0 };
+/** What a judgement metric expects: the verdicts set, or -- set before
+    Expected was offered -- the one rule there was, worse fails. */
+const expectedOf = (m        )           =>
+  (Array.isArray(m.expected) ? m.expected.map(String) : ["better", "same"]);
+/** The expected verdicts as a reader says them: "better or same". */
+const said = (values                   )         => {
+  const words = values.map((v) => v.toLowerCase());
+  return words.length < 2 ? words.join("") : `${words.slice(0, -1).join(", ")} or ${words.at(-1)}`;
+};
 
 // The families the Add metric picker lists the metrics under: what each reads.
-const TEXT = "The reply's text", RESULT = "The result", PROD = "Production's reply", GRADED = "Model-graded";
+const TEXT = "The reply's text", RESULT = "The result", RECORDED = "Recorded reply", GRADED = "Model-graded";
 
 const metrics                              = {
   equals: {
@@ -337,46 +375,54 @@ const metrics                              = {
         1 - d / Math.max(1, Math.max(got.length, want.length)));
     },
   },
-  // ---- against production ----------------------------------------------------
-  "equals-production": {
-    family: PROD,
-    label: "Same as production",
-    description: "Passes when the reply is what production replied to the same item.",
+  // ---- against the recorded reply --------------------------------------------
+  // n/a (null) when the target under test is the Recorded target itself: a
+  // reply compared with the reply it is says nothing (docs/workflow-sources.md).
+  "same-as-recorded": {
+    family: RECORDED,
+    parsedOnly: true,
+    label: "Same as recorded",
+    description: "Passes when the reply is what production recorded for the same item.",
     options: [],
     defaults: () => ({}),
-    score: (input) => {
-      const p = production(input).trim(), got = input.text.trim();
+    score: (input, _m, ctx) => {
+      if (ctx.recordedTarget) return null;
+      const p = recorded(input).trim(), got = input.text.trim();
       const a = json(got), b = json(p);
       const equal = !a.error && !b.error ? same(a.value, b.value) : got === p;
-      return ok(equal, equal ? "as production replied" : `production replied ${short(p)}`);
+      return ok(equal, equal ? "as recorded" : `recorded ${short(p)}`);
     },
   },
-  "fields-equal-production": {
-    family: PROD,
-    label: "Fields as production",
-    description: "Passes when the fields listed are what production's reply had.",
+  "fields-equal-recorded": {
+    family: RECORDED,
+    parsedOnly: true,
+    label: "Fields as recorded",
+    description: "Passes when the fields listed are what the recorded reply had.",
     options: [{ key: "fields", label: "Fields", type: "textarea" }],
     defaults: () => ({ fields: "" }),
     validate: (m, at, bad) => needs(m, ["fields"], at, bad),
-    score: (input, m) => {
-      const a = json(input.text), b = json(production(input));
+    score: (input, m, ctx) => {
+      if (ctx.recordedTarget) return null;
+      const a = json(input.text), b = json(recorded(input));
       if (a.error) return ok(false, `not JSON: ${a.error}`);
-      if (b.error) return ok(false, "production's reply was not JSON");
+      if (b.error) return ok(false, "the recorded reply was not JSON");
       const fields = lines(m.fields);
       const differ = fields.filter((f) => !same(atPath(a.value, f), atPath(b.value, f)));
-      return ok(!differ.length, differ.length ? `${differ.join(", ")} differ from production` : "the fields match production",
+      return ok(!differ.length, differ.length ? `${differ.join(", ")} differ from recorded` : "the fields match recorded",
         fields.length ? (fields.length - differ.length) / fields.length : 1);
     },
   },
-  "same-parse-outcome": {
-    family: PROD,
-    label: "Parses as production",
-    description: "Passes when the reply parses as JSON exactly when production's did.",
+  "same-parse-as-recorded": {
+    family: RECORDED,
+    parsedOnly: true,
+    label: "Parses as recorded",
+    description: "Passes when the reply parses as JSON exactly when the recorded reply did.",
     options: [],
     defaults: () => ({}),
-    score: (input) => {
-      const mine = !json(input.text).error, theirs = !json(production(input)).error;
-      return ok(mine === theirs, `${mine ? "parses" : "does not parse"}, production's ${theirs ? "parsed" : "did not"}`);
+    score: (input, _m, ctx) => {
+      if (ctx.recordedTarget) return null;
+      const mine = !json(input.text).error, theirs = !json(recorded(input)).error;
+      return ok(mine === theirs, `${mine ? "parses" : "does not parse"}, recorded ${theirs ? "parsed" : "did not"}`);
     },
   },
   // ---- model-graded -------------------------------------------------------------
@@ -385,43 +431,58 @@ const metrics                              = {
     label: "Rubric",
     description: "Asks the grader whether the reply meets the rubric.",
     graded: true,
-    options: [{ key: "rubric", label: "Rubric", type: "textarea" }, { key: "threshold", label: "Pass at", type: "number" }],
+    parsedOnly: true,
+    options: [{ key: "rubric", label: "Rubric", type: "textarea" }, { key: "threshold", label: "Expected: score at least", type: "number" }],
     defaults: () => ({ rubric: "", threshold: 0.5 }),
     validate: (m, at, bad) => needs(m, ["rubric"], at, bad),
-    score: async (input, m, ctx) => gradedPass(verdictOf(await ask(ctx,
-      `You are grading an output against a rubric.\n\nRubric:\n${str(m.rubric)}\n\nOutput:\n${input.text}\n\n${REPLY}`)),
-      num(m.threshold) ?? 0.5),
+    score: async (input, m, ctx) => gradedPass(await graded(ctx,
+      `You are grading an output against a rubric.\n\n${taskOf(input, "")}Rubric:\n${str(m.rubric)}\n\nOutput:\n${input.text}\n\n${REPLY}`,
+      VERDICT_SCHEMA), num(m.threshold) ?? 0.5),
   },
   factuality: {
     family: GRADED,
     label: "Factual",
-    description: "Asks the grader whether the reply agrees with the reference, or production's reply.",
+    description: "Asks the grader whether the reply agrees with the reference, or the recorded reply.",
     graded: true,
-    options: [{ key: "reference", label: "Reference", type: "textarea" }, { key: "threshold", label: "Pass at", type: "number" }],
+    parsedOnly: true,
+    options: [{ key: "reference", label: "Reference", type: "textarea" }, { key: "threshold", label: "Expected: score at least", type: "number" }],
     defaults: () => ({ reference: "", threshold: 0.5 }),
-    // A blank reference is production's reply.
-    score: async (input, m, ctx) => gradedPass(verdictOf(await ask(ctx,
+    // A blank reference is the recorded reply.
+    score: async (input, m, ctx) => gradedPass(await graded(ctx,
       `You are checking an output for factual consistency with a reference. Differences in wording or detail are `
-      + `fine; a contradiction, or a fact the reference does not support, is not.\n\nReference:\n`
-      + `${str(m.reference).trim() || production(input)}\n\nOutput:\n${input.text}\n\n${REPLY}`)), num(m.threshold) ?? 0.5),
+      + `fine; a contradiction, or a fact the reference does not support, is not.\n\n${taskOf(input, "")}Reference:\n`
+      + `${str(m.reference).trim() || recorded(input)}\n\nOutput:\n${input.text}\n\n${REPLY}`, VERDICT_SCHEMA), num(m.threshold) ?? 0.5),
   },
   "judge-vs-production": {
     family: GRADED,
-    label: "Judged against production",
-    description: "Asks the grader whether the reply is better than, the same as or worse than production's.",
+    label: "Judged against recorded",
+    description: "Asks the grader whether the reply is better than, the same as or worse than the recorded one.",
     graded: true,
-    options: [{ key: "task", label: "Task", type: "textarea" }],
-    defaults: () => ({ task: "" }),
+    parsedOnly: true,
+    expects: true,
+    options: [{ key: "expected", label: "Expected", type: "multi", choices: JUDGEMENTS },
+              { key: "task", label: "Additional guidance (optional)", type: "textarea" }],
+    defaults: () => ({ expected: ["better", "same"], task: "" }),
+    validate: (m, at, bad) => {
+      const known = JUDGEMENTS.map((j) => j.value);
+      if (m.expected !== undefined && (!Array.isArray(m.expected) || !m.expected.length
+          || m.expected.some((v) => !known.includes(String(v))))) {
+        bad.push(`${at}: Expected needs at least one of ${said(known)}`);
+      }
+    },
     score: async (input, m, ctx) => {
-      const v = verdictOf(await ask(ctx,
+      if (ctx.recordedTarget) return null;
+      const v = await graded(ctx,
         `Two outputs answer the same task. Say whether the NEW one is better than, the same as, or worse than the `
-        + `PRODUCTION one.${str(m.task).trim() ? `\n\nTask:\n${str(m.task)}` : ""}\n\nPRODUCTION:\n${production(input)}\n\n`
+        + `RECORDED one.\n\n${taskOf(input, m.task)}RECORDED:\n${recorded(input)}\n\n`
         + `NEW:\n${input.text}\n\nReply with one JSON object and nothing else: {"verdict": "better" or "same" or "worse", `
-        + `"reason": "one sentence"}.`));
-      const verdict = str(v.verdict).toLowerCase();
-      if (!["better", "same", "worse"].includes(verdict)) throw new Error(`the grader said ${short(JSON.stringify(v))}`);
-      return { pass: verdict !== "worse", score: verdict === "better" ? 1 : verdict === "same" ? 0.5 : 0,
-               reason: `${verdict}: ${str(v.reason)}` };
+        + `"reason": "one sentence"}.`,
+        { type: "object", additionalProperties: false, required: ["verdict", "reason"],
+          properties: { verdict: { type: "string", enum: JUDGEMENTS.map((j) => j.value) }, reason: { type: "string" } } },
+        (r) => str(r.verdict).toLowerCase() in JUDGEMENT_SCORE);
+      const verdict = str(v.verdict).toLowerCase(), expected = expectedOf(m);
+      return { pass: expected.includes(verdict), score: JUDGEMENT_SCORE[verdict] ,
+               reason: `${verdict} (expected ${said(expected)}): ${str(v.reason)}` };
     },
   },
 };

package/lab/run-evals.js

@@ -550,8 +550,25 @@ function callsFor(connections, links, text, plan, record) {
     if (core.STEP_TYPES[step?.type]?.asks === "request") {
       return () => send(links[k], c, step, plan.cells[k], record ?? textRecord(text));
     }
+    // The Recorded target replays each call's recorded reply, read as the flow
+    // reads one (like send, but sending nothing); it does not go through the
+    // model Read-as wrapper below -- the recorded body is read directly.
+    if (core.CONNECTION_TYPES[core.typeOf(c)]?.replays) {
+      return () => {
+        const r = record ?? textRecord(text);
+        const result = r && r.result;
+        if (!result || typeof result.status !== "number") return { raw: "", said: "", ms: 0, conn: null };
+        try {
+          const read = core.httpReplyOf(step, plan.cells[k], result.body, result.status,
+                                        { scope: r.scope || {}, now: r.at ?? null });
+          return { raw: read.raw, said: read.said, ms: 0, conn: null };
+        } catch (e) {
+          return { error: `the recorded reply could not be read as the flow reads it -- ${e.message}`, ms: 0, conn: null };
+        }
+      };
+    }
     const call = core.CONNECTION_TYPES[core.typeOf(c)]?.local
-      ? async sent => core.localAnswer(c, text, sent)
+      ? async sent => core.localAnswer(c, text, sent, record)
       : (sent, url) => ask(links[k], c, sent, url);
     if (!step?.readAs || !step?.step) return call;
     return async (sent, url) => {
@@ -588,8 +605,12 @@ function graderFor(run) {
       if (!c) return undefined;
       if (!made.has(ref.id)) {
         const link = reach(c, core.keyVar(ref.id, c.slug));
-        made.set(ref.id, async prompt => {
-          const r = await ask(link, { id: ref.id, ...c }, prompt, null);
+        made.set(ref.id, async (prompt, schema) => {
+          // Held to the metric's schema where the grader's API can hold a
+          // reply to one; a model or server that refuses the schema is asked
+          // again without it, the prompt alone saying the shape.
+          let r = await ask(link, { id: ref.id, ...c }, prompt, null, schema);
+          if (r.error && schema) r = await ask(link, { id: ref.id, ...c }, prompt, null);
           if (r.error) throw new Error(`the grader ${c.name || ref.id} failed: ${r.error}`);
           return r.raw ?? "";
         });
@@ -643,13 +664,13 @@ async function send(to, c, step, cell, record) {
   }
 }
 
-async function ask(to, c, prompt, dataUrl) {
+async function ask(to, c, prompt, dataUrl, schema = null) {
   const t0 = Date.now();
   const said = more => ({ ms: Date.now() - t0, conn: to.name, ...more });
   try {
     // The connection's type builds the whole request: the URL, the headers
     // (Anthropic's key travels in x-api-key), and the body.
-    const req = core.connectionRequest(c, prompt, dataUrl, to.base, to.key);
+    const req = core.connectionRequest(c, prompt, dataUrl, to.base, to.key, schema);
     const r = await fetch(req.url, {
       method: "POST",
       headers: { "Content-Type": "application/json", ...req.headers },
@@ -883,7 +904,10 @@ async function runSnapshot(o) {
   const plans = core.targetsOf(run).map((_, i) => {
     const { stages, tokens, connections, calls, cells } = core.stagesFor(run, i);
     const links = connections.map(c => reach(c, core.keyVar(c.id, c.slug)));
-    return { stages, tokens, connections, links, calls, cells };
+    // The Recorded target replays the recorded reply, so a metric that compares
+    // with the recorded reply has nothing to say of it (n/a).
+    const recordedTarget = connections.some(c => core.CONNECTION_TYPES[core.typeOf(c)]?.replays);
+    return { stages, tokens, connections, links, calls, cells, recordedTarget };
   });
 
   let results = [];
@@ -945,9 +969,9 @@ async function runSnapshot(o) {
       // A case is found by its file's name, whatever kind of file it is: a
       // text item a dataset grades is graded like an image.
       const kase = item.name ? graded.get(item.name) ?? null : null;
-      production = core.productionOf(run, record);
+      production = core.recordedReplyOf(run, record);
       sides.push({ res, scores: await core.itemScoresAsync(run, kase, res,
-        { production, grader: graderFor(run), group: gctx.resolve, item: item.name }) });
+        { production, grader: graderFor(run), group: gctx.resolve, item: item.name, recordedTarget: plan.recordedTarget }) });
     }
     // Production's reply to the item, where its record holds one: what a
     // metric compares with, kept so a re-score reads it again.