evals-lab 0.15.0 → 0.16.0

package/lab/kinds/list.mjs

@@ -454,7 +454,7 @@ export const FILTER_TYPES                                  = {
     },
   },
   "reject": {
-    label: "Reject", about: "Discard the reply when it matches its conditions.", icon: "trash",
+    label: "Reject", about: "Discard the response when it matches its conditions.", icon: "trash",
     matches: "reply", conditions: ["blank", "regex", "itemCount", "share", "items"], rejects: true,
     apply: (filter, reply, on) => ({ body: reply.body, items: reply.items, dropped: [],
       reject: rejectFires(filter.match, reply.body, on === "text" ? oneItem(reply.body) : reply.items) ? filter.id : null }),
@@ -833,7 +833,7 @@ const LIST_MODIFIERS                                       = {
 
 const LIST_KIND                         = {
   label: "List", noun: "items", offers: ["items"],
-  description: "Splits the reply into items (by commas, lines or a JSON array) that evals and later jobs read one by one.",
+  description: "Splits the response into items (by commas, lines or a JSON array) that evals and later jobs read one by one.",
   settings: [
     { key: "parse", label: "Parse as", type: "select", choices: [
       { value: "csv", label: "CSV" }, { value: "lines", label: "One a line" }, { value: "json", label: "JSON array" }] },

package/lab/metrics/builtin.mjs

@@ -37,7 +37,7 @@ export function atPath(v         , path        )          {
 
 /** Where [v] breaks [schema] -- the parts of JSON Schema a Parse JSON uses --
     or null where it holds. */
-export function schemaProblem(v         , schema         , at = "the reply")                {
+export function schemaProblem(v         , schema         , at = "the response")                {
   if (!schema || typeof schema !== "object") return null;
   const s = schema                           ;
   const type = s.type;
@@ -122,7 +122,7 @@ const spoken = (g          )              => (g.length === 1 ? g[0]  : g);
 
 /** The recorded reply, or the reason a metric that compares with it cannot. */
 const recorded = (input             )         => {
-  if (input.recorded == null) throw new Error("this item has no recorded reply to compare with");
+  if (input.recorded == null) throw new Error("this item has no recorded response to compare with");
   return input.recorded;
 };
 
@@ -132,7 +132,7 @@ const recorded = (input             )         => {
 function verdictOf(said        )                                                                           {
   const m = /\{[\s\S]*\}/.exec(said);
   const read = m ? json(m[0]) : { error: "no JSON" };
-  if (read.error || !read.value || typeof read.value !== "object") throw new Error(`the grader's reply could not be read: ${short(said)}`);
+  if (read.error || !read.value || typeof read.value !== "object") throw new Error(`the grader's response could not be read: ${short(said)}`);
   return read.value                           ;
 }
 /** The grader's verdict on [prompt], its reply held to [schema] where its API
@@ -203,13 +203,13 @@ const caseSaid = (m        )         => (m.ignoreCase === true ? "" : "matching
 const phrase = (...parts          )         => parts.filter(Boolean).join(", ");
 
 // The families the Add metric picker lists the metrics under: what each reads.
-const TEXT = "The reply's text", RESULT = "The result", RECORDED = "Recorded reply", GRADED = "Model-graded";
+const TEXT = "The response's text", RESULT = "The result", RECORDED = "Recorded response", GRADED = "Model-graded";
 
 const metrics                              = {
   equals: {
     family: TEXT,
     label: "Equals",
-    description: "Passes when the reply is exactly the value; as JSON, the same value with keys in any order.",
+    description: "Passes when the response is exactly the value; as JSON, the same value with keys in any order.",
     // As JSON, a reply equals the value as a value -- keys in any order;
     // otherwise, as the text, once trimmed.
     options: [{ key: "value", label: "Value", type: "textarea" }, { key: "json", label: "As JSON", type: "checkbox" }],
@@ -225,7 +225,7 @@ const metrics                              = {
   contains: {
     family: TEXT,
     label: "Contains",
-    description: "Passes when the reply contains the value; turned round, an exception excuses the value inside it.",
+    description: "Passes when the response contains the value; turned round, an exception excuses the value inside it.",
     options: [{ key: "value", label: "Value", type: "text" }, { key: "ignoreCase", label: "Ignore case", type: "checkbox" },
               { key: "except", label: "Except", type: "textarea" }],
     defaults: () => ({ value: "", ignoreCase: true }),
@@ -243,7 +243,7 @@ const metrics                              = {
   "contains-any": {
     family: TEXT,
     label: "Contains any",
-    description: "Passes when the reply contains at least one of the values, one a line.",
+    description: "Passes when the response contains at least one of the values, one a line.",
     options: [{ key: "values", label: "Values", type: "textarea" }, { key: "ignoreCase", label: "Ignore case", type: "checkbox" }],
     defaults: () => ({ values: "", ignoreCase: true }),
     says: (m) => phrase(quotedAll(lines(m.values)), caseSaid(m)),
@@ -258,7 +258,7 @@ const metrics                              = {
   "contains-all": {
     family: TEXT,
     label: "Contains all",
-    description: "Passes when the reply contains every value, one a line.",
+    description: "Passes when the response contains every value, one a line.",
     options: [{ key: "values", label: "Values", type: "textarea" }, { key: "ignoreCase", label: "Ignore case", type: "checkbox" }],
     defaults: () => ({ values: "", ignoreCase: true }),
     says: (m) => phrase(quotedAll(lines(m.values), "and"), caseSaid(m)),
@@ -274,7 +274,7 @@ const metrics                              = {
   regex: {
     family: TEXT,
     label: "Matches",
-    description: "Passes when the reply matches the regular expression.",
+    description: "Passes when the response matches the regular expression.",
     options: [{ key: "pattern", label: "Pattern", type: "text" }, { key: "flags", label: "Flags", type: "text" }],
     defaults: () => ({ pattern: "", flags: "" }),
     says: (m) => `/${str(m.pattern)}/${str(m.flags)}`,
@@ -287,7 +287,7 @@ const metrics                              = {
   "starts-with": {
     family: TEXT,
     label: "Starts with",
-    description: "Passes when the reply starts with the value.",
+    description: "Passes when the response starts with the value.",
     options: [{ key: "value", label: "Value", type: "text" }],
     defaults: () => ({ value: "" }),
     says: (m) => quoted(str(m.value)),
@@ -300,7 +300,7 @@ const metrics                              = {
   "is-json": {
     family: TEXT,
     label: "Is JSON",
-    description: "Passes when the reply is JSON and, with a schema, holds to it.",
+    description: "Passes when the response is JSON and, with a schema, holds to it.",
     options: [{ key: "schema", label: "Schema", type: "textarea" }],
     defaults: () => ({ schema: "" }),
     validate: (m, at, bad) => { if (str(m.schema).trim() && json(str(m.schema)).error) bad.push(`${at}: the schema is not JSON`); },
@@ -315,7 +315,7 @@ const metrics                              = {
   "json-field": {
     family: TEXT,
     label: "JSON field",
-    description: "Reads one field of a JSON reply and compares it.",
+    description: "Reads one field of a JSON response and compares it.",
     options: [
       { key: "path", label: "Field", type: "text" },
       { key: "op", label: "Is", type: "select", choices: [
@@ -348,7 +348,7 @@ const metrics                              = {
     family: RESULT,
     readsResult: true,
     label: "Discarded",
-    description: "Passes only when the job's rules discarded the reply.",
+    description: "Passes only when the job's rules discarded the response.",
     options: [],
     defaults: () => ({}),
     score: (input) => {
@@ -364,7 +364,7 @@ const metrics                              = {
     readsResult: true,
     readsRefusal: true,
     label: "Refused",
-    description: "Passes only when the target's call was refused: an error from the connection, not a reply.",
+    description: "Passes only when the variant's call was refused: an error from the connection, not a response.",
     options: [],
     defaults: () => ({}),
     score: (input) => {
@@ -413,7 +413,7 @@ const metrics                              = {
     family: RESULT,
     readsResult: true,
     label: "Latency",
-    description: "Passes when the reply came back within the time set.",
+    description: "Passes when the response came back within the time set.",
     options: [{ key: "max", label: "At most (ms)", type: "number" }],
     defaults: () => ({ max: 5000 }),
     // Unset, it is scored against 0 ms, which validate refuses: no number to say.
@@ -424,7 +424,7 @@ const metrics                              = {
   levenshtein: {
     family: TEXT,
     label: "Near",
-    description: "Passes when the reply is within a few edits of the value.",
+    description: "Passes when the response is within a few edits of the value.",
     options: [{ key: "value", label: "Value", type: "textarea" }, { key: "max", label: "Edits at most", type: "number" }],
     defaults: () => ({ value: "", max: 5 }),
     says: (m) => `within ${num(m.max) ?? 0} ${num(m.max) === 1 ? "edit" : "edits"} of ${quoted(str(m.value).trim())}`,
@@ -442,7 +442,7 @@ const metrics                              = {
     family: RECORDED,
     parsedOnly: true,
     label: "Same as recorded",
-    description: "Passes when the reply is what production recorded for the same item.",
+    description: "Passes when the response is what production recorded for the same item.",
     options: [],
     defaults: () => ({}),
     score: (input, _m, ctx) => {
@@ -457,7 +457,7 @@ const metrics                              = {
     family: RECORDED,
     parsedOnly: true,
     label: "Fields as recorded",
-    description: "Passes when the fields listed are what the recorded reply had.",
+    description: "Passes when the fields listed are what the recorded response had.",
     options: [{ key: "fields", label: "Fields", type: "textarea" }],
     defaults: () => ({ fields: "" }),
     says: (m) => lines(m.fields).join(", "),
@@ -466,7 +466,7 @@ const metrics                              = {
       if (ctx.recordedTarget) return null;
       const a = json(input.text), b = json(recorded(input));
       if (a.error) return ok(false, `not JSON: ${a.error}`);
-      if (b.error) return ok(false, "the recorded reply was not JSON");
+      if (b.error) return ok(false, "the recorded response was not JSON");
       const fields = lines(m.fields);
       const differ = fields.filter((f) => !same(atPath(a.value, f), atPath(b.value, f)));
       return ok(!differ.length, differ.length ? `${differ.join(", ")} differ from recorded` : "the fields match recorded",
@@ -477,7 +477,7 @@ const metrics                              = {
     family: RECORDED,
     parsedOnly: true,
     label: "Parses as recorded",
-    description: "Passes when the reply parses as JSON exactly when the recorded reply did.",
+    description: "Passes when the response parses as JSON exactly when the recorded response did.",
     options: [],
     defaults: () => ({}),
     score: (input, _m, ctx) => {
@@ -490,7 +490,7 @@ const metrics                              = {
   "llm-rubric": {
     family: GRADED,
     label: "Rubric",
-    description: "Asks the grader whether the reply meets the rubric.",
+    description: "Asks the grader whether the response meets the rubric.",
     graded: true,
     parsedOnly: true,
     options: [{ key: "rubric", label: "Rubric", type: "textarea" }, { key: "threshold", label: "Expected: score at least", type: "number" }],
@@ -504,12 +504,12 @@ const metrics                              = {
   factuality: {
     family: GRADED,
     label: "Factual",
-    description: "Asks the grader whether the reply agrees with the reference, or the recorded reply.",
+    description: "Asks the grader whether the response agrees with the reference, or the recorded response.",
     graded: true,
     parsedOnly: true,
     options: [{ key: "reference", label: "Reference", type: "textarea" }, { key: "threshold", label: "Expected: score at least", type: "number" }],
     defaults: () => ({ reference: "", threshold: 0.5 }),
-    says: (m) => phrase(str(m.reference).trim() ? `with ${quoted(str(m.reference).replace(/\s+/g, " ").trim())}` : "with the recorded reply",
+    says: (m) => phrase(str(m.reference).trim() ? `with ${quoted(str(m.reference).replace(/\s+/g, " ").trim())}` : "with the recorded response",
       num(m.threshold) != null && num(m.threshold) !== 0.5 ? `score at least ${num(m.threshold)}` : ""),
     // A blank reference is the recorded reply.
     score: async (input, m, ctx) => gradedPass(await graded(ctx,
@@ -520,7 +520,7 @@ const metrics                              = {
   "judge-vs-production": {
     family: GRADED,
     label: "Judged against recorded",
-    description: "Asks the grader whether the reply is better than, the same as or worse than the recorded one.",
+    description: "Asks the grader whether the response is better than, the same as or worse than the recorded one.",
     graded: true,
     parsedOnly: true,
     expects: true,

package/lab/run-evals.js

@@ -628,7 +628,7 @@ function callsFor(connections, links, text, plan, record) {
 // one, and the loop job 1's flow step sits in.
 const tokenRecord = (run, record, text) => {
   const r = record ?? (text != null ? textRecord(text) : null);
-  return r && { scope: r.scope || {}, at: r.at ?? null, loop: core.flowStepOf(run.jobs[0])?.loop ?? null };
+  return r && { scope: r.scope || {}, at: r.at ?? null, loop: core.flowStepOf(core.jobsOf(run)[0])?.loop ?? null };
 };
 
 // A prompt as a report restates it: an expression token reads an item's
@@ -768,27 +768,6 @@ function readUtf8(file) {
 const outcomesOf = (run, items, settled) =>
   core.targetsOf(run).map((_, i) => core.scenarioEvals(run, i, items, settled));
 
-// A link's Whole run verdict, per Target, keyed by the eval's id: a Library
-// group's `run` metrics read over the replies so far (readGroupRun). A private
-// group's Whole run is scenarioEvals' own (it holds the body); a link's body
-// is held apart, in --groups, so reading it beside the link's items is the
-// runner's (§17). [resolve] gives a link's body by its reference.
-function wholeRunsOf(run, items, resolve) {
-  const kind = core.lastKind(run);
-  return core.targetsOf(run).map((_, i) => {
-    const out = {};
-    const ress = (items || []).map(it => (it && !it.unrun ? it.scenarios?.[i]?.res : undefined));
-    for (const t of core.evalsOf(run)) {
-      // A private whole-run group is already scenarioEvals' verdict; here is
-      // the run beside a link's (or private group's) own items.
-      if (t.type !== "group" || core.isWholeRun(core.EVAL_TYPES.group, t)) continue;
-      const body = core.groupOf(t, { group: resolve });
-      if (body && Array.isArray(body.run) && body.run.length) out[t.id] = core.readGroupRun(body, ress, kind);
-    }
-    return out;
-  });
-}
-
 /**
  * One eval's verdict on one target, for the build: its own rule -- every
  * item it read passed, or a whole-run eval's verdict -- with [minPass] the
@@ -818,7 +797,7 @@ function evalVerdict(o, settled, minPass, wholeRun) {
 // Each target's evals, keyed by the eval's id: a whole-run eval's verdict, a
 // per-item eval's counts -- and, where a link reads a Whole run beside its
 // items, that verdict too -- with the build's verdict on each. [wholes] is
-// wholeRunsOf, one map per target.
+// core.wholeRunsOf, one map per target.
 const verdictsOf = (outcomes, settled, minPass, wholes = []) => outcomes.map((evals, i) =>
   Object.fromEntries(evals.map(o => {
     const wr = wholes[i] && wholes[i][o.id];
@@ -911,32 +890,30 @@ const scenariosOf = run => core.targetsOf(run).map((sc, i) => {
 async function runSnapshot(o) {
   const gctx = gradingContext(o);
   const run = readRun(o.run, gctx);
-  // One item a pipeline holds itself (Text, or Prompt only's bare one), or
-  // the Source's files.
-  const content = core.contentOf(run);
-  const inline = core.CONTENT_TYPES[content.type];
-  const bare = !!inline?.bare;
-  const text = inline?.inline && !bare ? inline.text?.(content, {}) ?? "" : "";
-  const files = content.type === "source" ? content.files || [] : [];
-  if (!bare && !files.length && !text.trim()) {
-    broken(`${o.run} names no files and no text, so there is nothing to run`);
-  }
+  // Each Get items step's items in turn: a Source's files, a text the
+  // pipeline holds itself, or Prompt only's bare one.
+  const listed = core.runItemsOf(run);
+  if (!listed.length) broken(`${o.run} names no files and no text, so there is nothing to run`);
+  // A row keeps the Get items step it came from when there are several, so
+  // Results groups its rows by Source; a run over one reads as it always did.
+  const several = core.itemContents(run).length > 1;
 
   // The one group a single-group run names, for an eval type of its own that
   // reads a case; a `group` eval reads each group's case by the item's name.
   const graded = gradedBy(gctx.primary());
 
-  const items = bare ? [{ name: null, kind: "text", text: null, bare: true }]
-    : text.trim() ? [{ name: null, kind: "text", text }]
-    : files.map(name => {
-        const ext = path.extname(name).toLowerCase();
-        if (RUN_IMAGE_TYPES.has(ext)) return { name, kind: "image" };
-        if (core.documentTypeOf(name)) return { name, kind: "document" };
-        // A record (flows/record.ts): one call of a Power Automate step, what
-        // an HTTP Request is rebuilt from.
-        if (ext === ".json") return { name, kind: "record" };
-        return { name, kind: "text", unmapped: !RUN_TEXT_TYPES.has(ext) };
-      });
+  const items = listed.map(({ name, text, bare, dir, step }) => {
+    const at = { dir, ...(several ? { step } : {}) };
+    if (bare) return { name: null, kind: "text", text: null, bare: true, ...at };
+    if (name == null) return { name: null, kind: "text", text, ...at };
+    const ext = path.extname(name).toLowerCase();
+    if (RUN_IMAGE_TYPES.has(ext)) return { name, kind: "image", ...at };
+    if (core.documentTypeOf(name)) return { name, kind: "document", ...at };
+    // A record (flows/record.ts): one call of a Power Automate step, what
+    // an HTTP Request is rebuilt from.
+    if (ext === ".json") return { name, kind: "record", ...at };
+    return { name, kind: "text", unmapped: !RUN_TEXT_TYPES.has(ext), ...at };
+  });
   const total = items.length;
   // Only the run's own image items need ImageMagick: a text-only Source runs
   // with nothing on PATH but the node running this.
@@ -987,8 +964,9 @@ async function runSnapshot(o) {
     let failed = null;
     for (const plan of plans) {
       let dataUrl = null;
+      // A later Get items' files are kept under its own folder (itemContents).
+      const file = item.name != null ? path.join(o.source, item.dir, item.name) : null;
       if (item.kind === "image") {
-        const file = path.join(o.source, item.name);
         if (!fs.existsSync(file)) { failed = `the file ${item.name} is not in the source`; break; }
         try {
           ({ dataUrl } = prepare(bin, file, plan.connections[0]));
@@ -998,13 +976,11 @@ async function runSnapshot(o) {
         }
       }
       if (item.kind === "document") {
-        const file = path.join(o.source, item.name);
         if (!fs.existsSync(file)) { failed = `the file ${item.name} is not in the source`; break; }
         dataUrl = documentOf(file);
       }
       let record = null;
       if (item.kind === "record") {
-        const file = path.join(o.source, item.name);
         try { record = JSON.parse(readUtf8(file)); } catch (e) {
           failed = fs.existsSync(file) ? `${item.name} is not a record -- ${e.message}` : `the file ${item.name} is not in the source`;
           break;
@@ -1012,12 +988,12 @@ async function runSnapshot(o) {
       }
       // A text file the Source does not hold is unrun, as an image is: the
       // run is incomplete rather than broken.
-      if (item.kind === "text" && !item.bare && item.text == null && !fs.existsSync(path.join(o.source, item.name))) {
+      if (item.kind === "text" && !item.bare && item.text == null && !fs.existsSync(file)) {
         failed = `the file ${item.name} is not in the source`;
         break;
       }
       const textOf = item.kind === "text" && !item.bare
-        ? (item.text != null ? item.text : readUtf8(path.join(o.source, item.name))) : null;
+        ? (item.text != null ? item.text : readUtf8(file)) : null;
       const res = await core.runPipeline(plan.stages, dataUrl, callsFor(plan.connections, plan.links, textOf, plan, record),
                                          { tokens: plan.tokens, text: textOf, record: tokenRecord(run, record, textOf) });
       // A case is found by its file's name, whatever kind of file it is: a
@@ -1029,7 +1005,8 @@ async function runSnapshot(o) {
     }
     // Production's reply to the item, where its record holds one: what a
     // metric compares with, kept so a re-score reads it again.
-    const out = { item: i, name: item.name, kind: item.kind, ...(production != null ? { production } : {}),
+    const out = { item: i, name: item.name, kind: item.kind, ...(item.step ? { getItems: item.step } : {}),
+                  ...(production != null ? { production } : {}),
                   unmapped: item.unmapped || null, ms: Date.now() - t0,
                   scenarios: sides, ...(failed ? { unrun: failed } : {}) };
     results[i] = out;
@@ -1044,7 +1021,7 @@ async function runSnapshot(o) {
   const complete = !cancelled && !unrun.length
     && ranItems.length === total && ranItems.every(it => it && !it.unrun);
   const outcomes = outcomesOf(run, ranItems, complete);
-  const wholes = wholeRunsOf(run, ranItems, gctx.resolve);
+  const wholes = core.wholeRunsOf(run, ranItems, gctx.resolve);
   const verdicts = verdictsOf(outcomes, complete, o.minPass, wholes);
   const report = {
     runner: RUNNER,
@@ -1139,7 +1116,7 @@ async function runRescore(o){
   // A stored run that stopped short re-scores as what it is: incomplete.
   const complete = items.every(it => isObject(it) && !it.unrun);
   const outcomes = outcomesOf(run, items, complete);
-  const wholes = wholeRunsOf(run, items, gctx.resolve);
+  const wholes = core.wholeRunsOf(run, items, gctx.resolve);
   const verdicts = verdictsOf(outcomes, complete, o.minPass, wholes);
   const report = {
     runner: RUNNER,
@@ -1172,17 +1149,18 @@ function cliRun(o, dataset) {
   // Graded, so the job answers with a list: the configuration that reads a
   // reply as this command always has (legacyTagsOut), under the dataset's
   // rules. A --pipeline states its own.
-  doc.jobs[0] = core.withOut(doc.jobs[0], core.legacyTagsOut(rulesOf(dataset), "LOWER"));
+  let job = core.withOut(core.jobsOf(doc)[0], core.legacyTagsOut(rulesOf(dataset), "LOWER"));
   if (o.tokens) {
     try {
-      doc.jobs[0] = core.withTokens(doc.jobs[0], JSON.parse(fs.readFileSync(o.tokens, "utf8")));
+      job = core.withTokens(job, JSON.parse(fs.readFileSync(o.tokens, "utf8")));
     } catch (e) {
       broken(`${o.tokens}: ${e.message}`);
     }
   }
+  doc = core.withJobs(doc, [job]);
   doc = core.withContent(doc, { type: "source", ref: { id: "cli", name: "" }, first: null, loops: 1, files: [] });
-  doc.targets = [{ id: "cli", name: "", connection: { id: "cli", name: "" }, profile: null,
-                   steps: [{ type: "prompt", prompt: o.prompt ?? "" }] }];
+  doc = core.withTargets(doc, [{ id: "cli", name: "", connection: { id: "cli", name: "" }, profile: null,
+                   steps: [{ type: "prompt", prompt: o.prompt ?? "" }] }]);
   doc.evals = [{ id: "cli", type: "group", name: "", continueOnFailure: true, group: { id: "cli", name: "" }, pin: null }];
   doc.profiles = { [core.answerKey("cli")]: { name: null, url: o.url, model: o.model, type: "openai-compatible" } };
   return doc;