evals-lab 0.6.0 → 0.7.0

package/lab/metrics/builtin.mjs

@@ -120,10 +120,10 @@ function holds(input             , m        , value        , except
 /** A requirement as Results names it: a term, or the group it was. */
 const spoken = (g          )              => (g.length === 1 ? g[0]  : g);
 
-/** Production's reply, or the reason a metric that compares with it cannot. */
-const production = (input             )         => {
-  if (input.production == null) throw new Error("this item has no production reply to compare with");
-  return input.production;
+/** The recorded reply, or the reason a metric that compares with it cannot. */
+const recorded = (input             )         => {
+  if (input.recorded == null) throw new Error("this item has no recorded reply to compare with");
+  return input.recorded;
 };
 
 // ---- model-graded -----------------------------------------------------------
@@ -147,7 +147,7 @@ const gradedPass = (v                              , threshold        )
 const REPLY = 'Reply with one JSON object and nothing else: {"pass": true or false, "score": a number from 0 to 1, "reason": "one sentence"}.';
 
 // The families the Add metric picker lists the metrics under: what each reads.
-const TEXT = "The reply's text", RESULT = "The result", PROD = "Production's reply", GRADED = "Model-graded";
+const TEXT = "The reply's text", RESULT = "The result", RECORDED = "Recorded reply", GRADED = "Model-graded";
 
 const metrics                              = {
   equals: {
@@ -337,46 +337,51 @@ const metrics                              = {
         1 - d / Math.max(1, Math.max(got.length, want.length)));
     },
   },
-  // ---- against production ----------------------------------------------------
-  "equals-production": {
-    family: PROD,
-    label: "Same as production",
-    description: "Passes when the reply is what production replied to the same item.",
+  // ---- against the recorded reply --------------------------------------------
+  // n/a (null) when the target under test is the Recorded target itself: a
+  // reply compared with the reply it is says nothing (docs/workflow-sources.md).
+  "same-as-recorded": {
+    family: RECORDED,
+    label: "Same as recorded",
+    description: "Passes when the reply is what production recorded for the same item.",
     options: [],
     defaults: () => ({}),
-    score: (input) => {
-      const p = production(input).trim(), got = input.text.trim();
+    score: (input, _m, ctx) => {
+      if (ctx.recordedTarget) return null;
+      const p = recorded(input).trim(), got = input.text.trim();
       const a = json(got), b = json(p);
       const equal = !a.error && !b.error ? same(a.value, b.value) : got === p;
-      return ok(equal, equal ? "as production replied" : `production replied ${short(p)}`);
+      return ok(equal, equal ? "as recorded" : `recorded ${short(p)}`);
     },
   },
-  "fields-equal-production": {
-    family: PROD,
-    label: "Fields as production",
-    description: "Passes when the fields listed are what production's reply had.",
+  "fields-equal-recorded": {
+    family: RECORDED,
+    label: "Fields as recorded",
+    description: "Passes when the fields listed are what the recorded reply had.",
     options: [{ key: "fields", label: "Fields", type: "textarea" }],
     defaults: () => ({ fields: "" }),
     validate: (m, at, bad) => needs(m, ["fields"], at, bad),
-    score: (input, m) => {
-      const a = json(input.text), b = json(production(input));
+    score: (input, m, ctx) => {
+      if (ctx.recordedTarget) return null;
+      const a = json(input.text), b = json(recorded(input));
       if (a.error) return ok(false, `not JSON: ${a.error}`);
-      if (b.error) return ok(false, "production's reply was not JSON");
+      if (b.error) return ok(false, "the recorded reply was not JSON");
       const fields = lines(m.fields);
       const differ = fields.filter((f) => !same(atPath(a.value, f), atPath(b.value, f)));
-      return ok(!differ.length, differ.length ? `${differ.join(", ")} differ from production` : "the fields match production",
+      return ok(!differ.length, differ.length ? `${differ.join(", ")} differ from recorded` : "the fields match recorded",
         fields.length ? (fields.length - differ.length) / fields.length : 1);
     },
   },
-  "same-parse-outcome": {
-    family: PROD,
-    label: "Parses as production",
-    description: "Passes when the reply parses as JSON exactly when production's did.",
+  "same-parse-as-recorded": {
+    family: RECORDED,
+    label: "Parses as recorded",
+    description: "Passes when the reply parses as JSON exactly when the recorded reply did.",
     options: [],
     defaults: () => ({}),
-    score: (input) => {
-      const mine = !json(input.text).error, theirs = !json(production(input)).error;
-      return ok(mine === theirs, `${mine ? "parses" : "does not parse"}, production's ${theirs ? "parsed" : "did not"}`);
+    score: (input, _m, ctx) => {
+      if (ctx.recordedTarget) return null;
+      const mine = !json(input.text).error, theirs = !json(recorded(input)).error;
+      return ok(mine === theirs, `${mine ? "parses" : "does not parse"}, recorded ${theirs ? "parsed" : "did not"}`);
     },
   },
   // ---- model-graded -------------------------------------------------------------
@@ -395,27 +400,27 @@ const metrics                              = {
   factuality: {
     family: GRADED,
     label: "Factual",
-    description: "Asks the grader whether the reply agrees with the reference, or production's reply.",
+    description: "Asks the grader whether the reply agrees with the reference, or the recorded reply.",
     graded: true,
     options: [{ key: "reference", label: "Reference", type: "textarea" }, { key: "threshold", label: "Pass at", type: "number" }],
     defaults: () => ({ reference: "", threshold: 0.5 }),
-    // A blank reference is production's reply.
+    // A blank reference is the recorded reply.
     score: async (input, m, ctx) => gradedPass(verdictOf(await ask(ctx,
       `You are checking an output for factual consistency with a reference. Differences in wording or detail are `
       + `fine; a contradiction, or a fact the reference does not support, is not.\n\nReference:\n`
-      + `${str(m.reference).trim() || production(input)}\n\nOutput:\n${input.text}\n\n${REPLY}`)), num(m.threshold) ?? 0.5),
+      + `${str(m.reference).trim() || recorded(input)}\n\nOutput:\n${input.text}\n\n${REPLY}`)), num(m.threshold) ?? 0.5),
   },
   "judge-vs-production": {
     family: GRADED,
-    label: "Judged against production",
-    description: "Asks the grader whether the reply is better than, the same as or worse than production's.",
+    label: "Judged against recorded",
+    description: "Asks the grader whether the reply is better than, the same as or worse than the recorded one.",
     graded: true,
     options: [{ key: "task", label: "Task", type: "textarea" }],
     defaults: () => ({ task: "" }),
     score: async (input, m, ctx) => {
       const v = verdictOf(await ask(ctx,
         `Two outputs answer the same task. Say whether the NEW one is better than, the same as, or worse than the `
-        + `PRODUCTION one.${str(m.task).trim() ? `\n\nTask:\n${str(m.task)}` : ""}\n\nPRODUCTION:\n${production(input)}\n\n`
+        + `RECORDED one.${str(m.task).trim() ? `\n\nTask:\n${str(m.task)}` : ""}\n\nRECORDED:\n${recorded(input)}\n\n`
         + `NEW:\n${input.text}\n\nReply with one JSON object and nothing else: {"verdict": "better" or "same" or "worse", `
         + `"reason": "one sentence"}.`));
       const verdict = str(v.verdict).toLowerCase();

package/lab/run-evals.js

@@ -550,8 +550,25 @@ function callsFor(connections, links, text, plan, record) {
     if (core.STEP_TYPES[step?.type]?.asks === "request") {
       return () => send(links[k], c, step, plan.cells[k], record ?? textRecord(text));
     }
+    // The Recorded target replays each call's recorded reply, read as the flow
+    // reads one (like send, but sending nothing); it does not go through the
+    // model Read-as wrapper below -- the recorded body is read directly.
+    if (core.CONNECTION_TYPES[core.typeOf(c)]?.replays) {
+      return () => {
+        const r = record ?? textRecord(text);
+        const result = r && r.result;
+        if (!result || typeof result.status !== "number") return { raw: "", said: "", ms: 0, conn: null };
+        try {
+          const read = core.httpReplyOf(step, plan.cells[k], result.body, result.status,
+                                        { scope: r.scope || {}, now: r.at ?? null });
+          return { raw: read.raw, said: read.said, ms: 0, conn: null };
+        } catch (e) {
+          return { error: `the recorded reply could not be read as the flow reads it -- ${e.message}`, ms: 0, conn: null };
+        }
+      };
+    }
     const call = core.CONNECTION_TYPES[core.typeOf(c)]?.local
-      ? async sent => core.localAnswer(c, text, sent)
+      ? async sent => core.localAnswer(c, text, sent, record)
       : (sent, url) => ask(links[k], c, sent, url);
     if (!step?.readAs || !step?.step) return call;
     return async (sent, url) => {
@@ -883,7 +900,10 @@ async function runSnapshot(o) {
   const plans = core.targetsOf(run).map((_, i) => {
     const { stages, tokens, connections, calls, cells } = core.stagesFor(run, i);
     const links = connections.map(c => reach(c, core.keyVar(c.id, c.slug)));
-    return { stages, tokens, connections, links, calls, cells };
+    // The Recorded target replays the recorded reply, so a metric that compares
+    // with the recorded reply has nothing to say of it (n/a).
+    const recordedTarget = connections.some(c => core.CONNECTION_TYPES[core.typeOf(c)]?.replays);
+    return { stages, tokens, connections, links, calls, cells, recordedTarget };
   });
 
   let results = [];
@@ -945,9 +965,9 @@ async function runSnapshot(o) {
       // A case is found by its file's name, whatever kind of file it is: a
       // text item a dataset grades is graded like an image.
       const kase = item.name ? graded.get(item.name) ?? null : null;
-      production = core.productionOf(run, record);
+      production = core.recordedReplyOf(run, record);
       sides.push({ res, scores: await core.itemScoresAsync(run, kase, res,
-        { production, grader: graderFor(run), group: gctx.resolve, item: item.name }) });
+        { production, grader: graderFor(run), group: gctx.resolve, item: item.name, recordedTarget: plan.recordedTarget }) });
     }
     // Production's reply to the item, where its record holds one: what a
     // metric compares with, kept so a re-score reads it again.