evals-lab 0.10.0 → 0.11.0

package/lab/flows/record.mjs

@@ -8,7 +8,10 @@
 import { evaluate,               } from "./wdl.mjs";
 
 export const RECORD_FORMAT = "evals-lab/record";
-export const RECORD_VERSION = 1;
+export const RECORD_VERSION = 2;
+/** The origins a record names. Version 2 added `edited`: a call changed in the
+    Call dialog, keeping the `run` and `iteration` it came from. */
+export const RECORD_ORIGINS = ["run", "hand", "upload", "edited"]         ;
 
 /** What a step's template read, by where the expression reads it from. */
                               
@@ -39,11 +42,20 @@ export const RECORD_VERSION = 1;
                                                       
                                                                  
                               
-                                                                                
+                                                                                      
  
 
 const isObj = (v         )                               => v != null && typeof v === "object" && !Array.isArray(v);
 
+/** A version-1 record as today's, as every reader takes one (server.py's
+    `upgrade_record`, its twin). Version 2 only widened the origins, so the
+    upgrade is the number; anything else is returned as it is, for
+    recordProblems to judge. */
+export function upgradeRecord(value         )          {
+  return isObj(value) && value.format === RECORD_FORMAT && value.version === 1
+    ? { ...value, version: RECORD_VERSION } : value;
+}
+
 /** Every reason [value] is not a record this lab keeps, as sentences. */
 export function recordProblems(value         )           {
   if (!isObj(value)) return ["a record is a JSON object"];
@@ -68,8 +80,8 @@ export function recordProblems(value         )           {
     bad.push("a record's result is { status, body }, { error } or null");
   }
   const origin = value.origin;
-  if (!isObj(origin) || !["run", "hand", "upload"].includes(origin.kind          )) {
-    bad.push("a record's origin is run, hand or upload");
+  if (!isObj(origin) || !(RECORD_ORIGINS                      ).includes(origin.kind)) {
+    bad.push("a record's origin is run, hand, upload or edited");
   }
   return bad;
 }

package/lab/kinds/list.mjs

@@ -58,7 +58,7 @@ export const PARSE_FORMATS = ["csv", "lines", "json"]         ;
 
 /** A reply as the text its items come from, and the items, each trimmed and
     none blank. Nothing else is done to them: cleaning is the filters' -- a
-    before-Read-as text filter shapes the reply, this splits it (#405). */
+    Pre-format text filter shapes the reply, this splits it (#405). */
 export function splitItems(reply        , parse              = "csv")                                    {
   const body = String(reply ?? "");
   if (parse === "json") {
@@ -288,7 +288,12 @@ export const FILTER_SEPARATORS = ["comma", "lines", "semicolon", "none"]
 /** What applying a filter or a set left: the reply text (a Rewrite text filter
     changes it), the items kept, what was dropped and by which filter, and --
     the first filter that rejected -- the reply's rejection (null when none). */
-                                                                                                           
+                                
+                                                                           
+                                                                            
+                                        
+                   
+ 
 
 /** A base filter type (#334's decision: a registry, not an enum). A reader
     asks the entry by `filter.type`; nothing branches on the id. */
@@ -296,8 +301,10 @@ export const FILTER_SEPARATORS = ["comma", "lines", "semicolon", "none"]
                                                 
                 
                                                                            
-                                                                               
-                                                                         
+                 
+                                                                           
+                                                                           
+                                                                       
                 
                                              
                                                                             
@@ -350,22 +357,22 @@ function rewriteText(op                                     )
 
 /** The lab's base filter types (#405). Generic code asks this registry by
     `filter.type`; it never names an id. `rewrite-text`/`reject-text` read the
-    raw reply before it is read into items (the before-Read-as stage); the rest
-    read the items after (the after-Read-as stage). */
+    raw reply before it is read into items (the Pre-format stage); the rest
+    read the items after (the Post-format stage). */
 export const FILTER_TYPES                                  = {
   "rewrite-text": {
-    label: "Rewrite text", scope: "text", conditions: [],
+    label: "Rewrite text", about: "Remove thinking, a preamble or a pattern from the reply's text.", scope: "text", conditions: [],
     opProblems: (f, at) => opKindProblems(f.op, at, {
       thinking: [], preamble: [], replace: ["pattern", "replacement"],
     }),
     apply: (filter, reply) => ({ body: rewriteText(filter.op)(reply.body), items: reply.items, dropped: [], reject: null }),
   },
   "reject-text": {
-    label: "Reject text", scope: "text", conditions: ["blank", "regex"], rejects: true,
+    label: "Reject text", about: "Discard the reply when it is blank or matches a pattern.", scope: "text", conditions: ["blank", "regex"], rejects: true,
     apply: (filter, reply) => ({ body: reply.body, items: reply.items, dropped: [], reject: rejectTextHolds(filter.match, reply.body) ? filter.id : null }),
   },
   "drop-item": {
-    label: "Drop item", scope: "item", conditions: ITEM_CONDITIONS,
+    label: "Drop item", about: "Remove the items that match its conditions.", scope: "item", conditions: ITEM_CONDITIONS,
     apply(filter, reply) {
       const test = matcher(filter.match);
       if (!test) return outcome(reply);
@@ -380,7 +387,7 @@ export const FILTER_TYPES                                  = {
     },
   },
   "rewrite-item": {
-    label: "Rewrite item", scope: "item", conditions: MATCHER_KEYS,
+    label: "Rewrite item", about: "Tidy each item, or change its case.", scope: "item", conditions: MATCHER_KEYS,
     opProblems: (f, at) => opKindProblems(f.op, at, { tidy: [], case: ["mode"] }, op =>
       op.kind === "case" && !CASE_MODES.includes(op.mode          ) ? [`${at}'s case needs a mode`] : []),
     apply(filter, reply) {
@@ -393,7 +400,7 @@ export const FILTER_TYPES                                  = {
     },
   },
   "keep-first": {
-    label: "Keep first", scope: "list", conditions: [],
+    label: "Keep first", about: "Keep the first N items.", scope: "list", conditions: [],
     opProblems: (f, at) => (Number.isInteger(f.op?.n) && (f.op .n          ) >= 1
       ? extraOpKeys(f.op, ["n"], at) : [`${at} needs a whole number of 1 or more`]),
     apply(filter, reply) {
@@ -402,7 +409,7 @@ export const FILTER_TYPES                                  = {
     },
   },
   "reject-reply": {
-    label: "Reject reply", scope: "answer", conditions: ["blank", "itemCount", "share", "items"], rejects: true,
+    label: "Reject reply", about: "Discard the whole reply when its items match its conditions.", scope: "answer", conditions: ["blank", "itemCount", "share", "items"], rejects: true,
     apply: (filter, reply) => ({ body: reply.body, items: reply.items, dropped: [], reject: rejectHolds(filter.match, reply.body, reply.items) ? filter.id : null }),
   },
 };
@@ -459,22 +466,24 @@ export function applyFilter(filter        , reply             )                {
     reshapes the text the next reads, each item filter narrows or rewrites the
     items, and the first filter that rejects gates the answer and stops the
     rest. Given a [stage], only the filters whose type runs there apply: a set
-    belongs to no stage (#432), so one linked both before and after Read as runs
+    belongs to no stage (#432), so one linked both Pre-format and Post-format runs
     its text filters on the raw reply and the rest on the items. Pure. */
 export function applyFilterSet(filters                   , reply             , stage                   )                {
   const instruction = reply.instruction ?? "";
   let body = reply.body, items = reply.items;
   const dropped            = [];
+  let emptied                    ;
   for (const filter of filters) {
     const scope = FILTER_TYPES[filter.type]?.scope;
     if (stage && scope && (scope === "text") !== (stage === "text")) continue;
     const out = applyFilter(filter, { body, items, instruction });
     body = out.body;
+    if (items.length && !out.items.length) emptied = filter.id;
     items = out.items;
     dropped.push(...out.dropped);
     if (out.reject != null) return { body, items, dropped, reject: out.reject };
   }
-  return { body, items, dropped, reject: null };
+  return { body, items, dropped, reject: null, ...(emptied ? { emptied } : {}) };
 }
 
 // A newline always ends an item, as it does in the real csv/lines parse
@@ -639,16 +648,16 @@ export function applyCase(item        , mode        )         {
   }
 }
 
-/** The id of the synthetic modifier a linked or private after-Read-as filter
+/** The id of the synthetic modifier a linked or private Post-format filter
     set applies under at run time (#339, #405): it is never in a document or
     the Add menu -- a job carries a `filters` step, which `stageModifiers`
     turns into one of these where the step sits, so a set cleans and gates a
-    list reply exactly as it reads. The before-Read-as (text) stage runs its
+    list reply exactly as it reads. The Pre-format (text) stage runs its
     filters on the raw reply before it is read, not as a modifier. */
 export const FILTER_MODIFIER = "filterSet";
 
 const LIST_MODIFIERS                                       = {
-  // An after-Read-as filter set's filters, applied where its link sits: the
+  // A Post-format filter set's filters, applied where its link sits: the
   // run's only reader of a resolved items-stage set, hidden from the Add menu
   // (a set is linked in Responses, not added as a modifier). The lab's own
   // cleaning modifiers are retired (#405) -- every cleaning step is a filter.
@@ -657,8 +666,8 @@ const LIST_MODIFIERS                                       = {
     apply(items, m, ctx) {
       const out = applyFilterSet((m.filters            ) || [], { body: ctx.body, items, instruction: ctx.instruction }, "items");
       return out.reject != null
-        ? { value: out.items, dropped: out.dropped, reject: out.reject }
-        : { value: out.items, dropped: out.dropped };
+        ? { value: out.items, dropped: out.dropped, reject: out.reject, by: out.reject }
+        : { value: out.items, dropped: out.dropped, ...(out.emptied ? { by: out.emptied } : {}) };
     },
   },
 };
@@ -703,7 +712,7 @@ export const CLEANING_MODIFIERS = ["reject", "drop", "tidy", "drop-echoes", "ded
 const baseFilter = (id        , type        , extra                  = {})         =>
   ({ id, type, match: {}, options: { ignoreFailures: false, repeat: 1 }, tests: [], ...extra });
 
-/** The after-Read-as filter(s) a retired cleaning modifier [m] becomes, in the
+/** The Post-format filter(s) a retired cleaning modifier [m] becomes, in the
     same order and reading a reply the same way: Tidy and Case are a Rewrite
     item, Drop prompt echoes and Remove duplicates a Drop item on the `echoes`
     and `duplicate` conditions, Keep first its own type, and Drop items / Reject
@@ -722,7 +731,7 @@ export function cleaningFilters(m          )           {
   }
 }
 
-/** The before-Read-as (text) filters the List `preamble` setting becomes:
+/** The Pre-format (text) filters the List `preamble` setting becomes:
     Remove thinking then Remove preamble, which together read a reply exactly as
     `preamble: true` did (#405). */
 export function preambleFilters()           {
@@ -777,8 +786,8 @@ function legacyTagsCleaning(rules                 , mode        )
 
 /** What a v2 pipeline's `tags` job is upgraded to, and the runtime List
     configuration the checks hold to that kind's answers (parity-check.js): the
-    reply read by the before-Read-as text filters (`before`) then split, its
-    items cleaned and gated by the after-Read-as filters, carried as the
+    reply read by the Pre-format text filters (`before`) then split, its
+    items cleaned and gated by the Post-format filters, carried as the
     synthetic `FILTER_MODIFIER` modifier so a caller runs them as a run would.
     Derived from `legacyTagsCleaning` through `cleaningFilters`, the same path
     the version-15 upgrade takes, so they cannot disagree. */

package/lab/run-evals.js

@@ -633,7 +633,7 @@ function graderFor(run) {
       const c = run.profiles?.[ref.id];
       if (!c) return undefined;
       if (!made.has(ref.id)) {
-        const link = reach(c, core.keyVar(ref.id, c.slug));
+        const link = reach(c, core.keyVar(core.answerConnectionId(ref.id, c), c.slug));
         made.set(ref.id, async (prompt, schema) => {
           // Held to the metric's schema where the grader's API can hold a
           // reply to one; a model or server that refuses the schema is asked
@@ -1160,10 +1160,10 @@ function cliRun(o, dataset) {
     }
   }
   doc = core.withContent(doc, { type: "source", ref: { id: "cli", name: "" }, first: null, loops: 1, files: [] });
-  doc.targets = [{ id: "cli", name: "", profile: { id: "cli", name: "" },
+  doc.targets = [{ id: "cli", name: "", connection: { id: "cli", name: "" }, profile: null,
                    steps: [{ type: "prompt", prompt: o.prompt ?? "" }] }];
   doc.evals = [{ id: "cli", type: "group", name: "", continueOnFailure: true, group: { id: "cli", name: "" }, pin: null }];
-  doc.profiles = { cli: { name: null, url: o.url, model: o.model, type: "openai-compatible" } };
+  doc.profiles = { [core.answerKey("cli")]: { name: null, url: o.url, model: o.model, type: "openai-compatible" } };
   return doc;
 }