evals-lab 0.1.3 → 0.2.0

package/lab/run-evals.js

@@ -453,21 +453,47 @@ let seconds = REQUEST_CAP; // the run's per-request cap, set in main()
 // or, over Prompt only, the prompt -- and the stage reads it as it would a
 // model's.
 //
-// A call that asks for a whole request (an HTTP Request, #flows) builds it
-// from its step, the scenario's cell and the item's record, and is handed
-// only its words: the transport is where the three meet.
+// A target step that asks for a whole request (an HTTP Request, #flows)
+// builds it from the step, the job's flow step and the item's record, and is
+// handed only its words: the transport is where the three meet.
+//
+// Any other step's reply, in a job that reads the flow's step through a Read
+// as, is read the same way: put in the body the flow's API would have sent
+// it in, then read as the flow reads it -- so a model asked in words and
+// production are compared alike (docs/pipeline-model.md §16 › Targets).
 function callsFor(connections, links, text, plan, record) {
   return connections.map((c, k) => {
     const step = plan?.calls?.[k];
     if (core.STEP_TYPES[step?.type]?.asks === "request") {
       return () => send(links[k], c, step, plan.cells[k], record ?? textRecord(text));
     }
-    return core.CONNECTION_TYPES[core.typeOf(c)]?.local
+    const call = core.CONNECTION_TYPES[core.typeOf(c)]?.local
       ? async sent => core.localAnswer(c, text, sent)
       : (sent, url) => ask(links[k], c, sent, url);
+    if (!step?.readAs || !step?.step) return call;
+    return async (sent, url) => {
+      const r = await call(sent, url);
+      if (r.error || r.raw == null) return r;
+      try {
+        return { ...r, ...core.readFlowReply(step, r.raw, record ?? textRecord(text)) };
+      } catch (e) {
+        return { ...r, error: `the reply could not be read as the flow reads it -- ${e.message}` };
+      }
+    };
   });
 }
 
+// What an expression token reads: the item's record, or a text item read as
+// one, and the loop job 1's flow step sits in.
+const tokenRecord = (run, record, text) => {
+  const r = record ?? (text != null ? textRecord(text) : null);
+  return r && { scope: r.scope || {}, at: r.at ?? null, loop: core.flowStepOf(run.jobs[0])?.loop ?? null };
+};
+
+// A prompt as a report restates it: an expression token reads an item's
+// record, and a report has none to name, so its words stay as written.
+const shown = (text, tokens) => { try { return core.resolvePrompt(text, tokens); } catch { return text; } };
+
 /** A Setup profile the run carries, asked as a grader: its reply's text, or
     the failure as an error the metric reports. One transport per profile. */
 const graders = new WeakMap();
@@ -590,7 +616,7 @@ function readUtf8(file) {
 // the page reads it: a whole-run test's verdict, settled once every item is
 // in, and a per-item test's counts. [settled] is false for a run that
 // stopped short, whose whole-run tests have not settled.
-const verdictsOf = (run, items, settled) => run.scenarios.map((_, i) =>
+const verdictsOf = (run, items, settled) => core.targetsOf(run).map((_, i) =>
   Object.fromEntries(core.scenarioTests(run, i, items, settled).map(o => [o.id, {
     name: o.label, skipped: o.skipped,
     ...(o.whole
@@ -600,7 +626,7 @@ const verdictsOf = (run, items, settled) => run.scenarios.map((_, i) =>
 
 // The run as the report restates it: each scenario's stages with the
 // connection each one asked.
-const scenariosOf = run => run.scenarios.map((sc, i) => {
+const scenariosOf = run => core.targetsOf(run).map((sc, i) => {
   const { stages, connections } = core.stagesFor(run, i);
   return {
     n: i + 1, name: sc.name || null,
@@ -649,7 +675,7 @@ async function runSnapshot(o) {
   // Each scenario once: its stages, the jobs' token sets, and a transport
   // per stage to the profile that stage resolves to, keyed by that
   // profile's id.
-  const plans = run.scenarios.map((_, i) => {
+  const plans = core.targetsOf(run).map((_, i) => {
     const { stages, tokens, connections, calls, cells } = core.stagesFor(run, i);
     const links = connections.map(c => reach(c, core.keyVar(c.id)));
     return { stages, tokens, connections, links, calls, cells };
@@ -704,7 +730,7 @@ async function runSnapshot(o) {
       const textOf = item.kind === "text" && !item.bare
         ? (item.text != null ? item.text : readUtf8(path.join(o.source, item.name))) : null;
       const res = await core.runPipeline(plan.stages, dataUrl, callsFor(plan.connections, plan.links, textOf, plan, record),
-                                         { tokens: plan.tokens, text: textOf });
+                                         { tokens: plan.tokens, text: textOf, record: tokenRecord(run, record, textOf) });
       // A case is found by its file's name, whatever kind of file it is: a
       // text item a dataset grades is graded like an image.
       const kase = item.name ? graded.get(item.name) ?? null : null;
@@ -822,14 +848,14 @@ function cliRun(o, dataset) {
   doc.jobs[0] = core.withOut(doc.jobs[0], core.legacyTagsOut(rulesOf(dataset), "LOWER"));
   if (o.tokens) {
     try {
-      doc.jobs[0] = core.withCall(doc.jobs[0], { tokenMappings: JSON.parse(fs.readFileSync(o.tokens, "utf8")) });
+      doc.jobs[0] = core.withTokens(doc.jobs[0], JSON.parse(fs.readFileSync(o.tokens, "utf8")));
     } catch (e) {
       broken(`${o.tokens}: ${e.message}`);
     }
   }
   doc = core.withContent(doc, { type: "source", ref: { id: "cli", name: "" }, first: null, loops: 1, files: [] });
-  doc.scenarios = [{ id: "cli", name: "", profile: { id: "cli", name: "" },
-                     stages: [{ prompt: o.prompt ?? "" }] }];
+  doc.targets = [{ id: "cli", name: "", profile: { id: "cli", name: "" },
+                   steps: [{ type: "prompt", prompt: o.prompt ?? "" }] }];
   doc.tests = [{ id: "cli", type: "metrics", name: "", continueOnFailure: true, dataset: { id: "cli", name: "" },
                  mode: "all", threshold: null, grader: null, over: "item", metrics: [{ type: "case" }] }];
   doc.profiles = { cli: { name: null, url: o.url, model: o.model, type: "openai-compatible" } };
@@ -881,7 +907,7 @@ async function main() {
   }
   if (o.run) {
     if (o.pipeline || o.prompt != null || o.tokens || o.model || o.url || o.replies || o.samples || o.files) {
-      broken("--run names everything a run takes -- scenarios, files and text -- so it "
+      broken("--run names everything a run takes -- targets, files and text -- so it "
            + "cannot be given with --pipeline, --prompt, --tokens, --model, --url, --replies, --samples or --files.");
     }
     const n = Number(o.timeout ?? REQUEST_CAP);
@@ -920,8 +946,8 @@ async function main() {
   }
   const run = o.pipeline ? readRun(o.pipeline, dataset) : cliRun(o, dataset);
   if (o.pipeline) {
-    if (run.scenarios.length !== 1) {
-      broken(`${o.pipeline} has ${run.scenarios.length} scenarios, and --pipeline grades one `
+    if (core.targetsOf(run).length !== 1) {
+      broken(`${o.pipeline} has ${core.targetsOf(run).length} targets, and --pipeline grades one `
            + "against the set -- --run runs them all.");
     }
     if (!core.testsDataset(run)) {
@@ -934,7 +960,7 @@ async function main() {
   // Case by case: each item against its case, as the case metric of the
   // test that names the dataset reads it (scoreCase).
   const { stages, tokens, connections } = core.stagesFor(run, 0);
-  const prompt = core.resolvePrompt(stages[0].text, core.tokenSet(tokens, 0));
+  const prompt = shown(stages[0].text, core.tokenSet(tokens, 0));
 
   // The graded half, through the function the tab builds its own list with.
   const set = core.gradedSetFrom(dataset);
@@ -1170,7 +1196,7 @@ async function main() {
       stages: stages.map((s, i) => {
         const { id, ...connection } = connections[i];
         return { n: i + 1, kind: s.kind, withImage: s.withImage,
-                 prompt: core.resolvePrompt(s.text, core.tokenSet(tokens, i)), profile: id, connection };
+                 prompt: shown(s.text, core.tokenSet(tokens, i)), profile: id, connection };
       }),
     } } : {}),
     // Which variable a key came from, never the key.