evals-lab 0.12.0 → 0.14.0

package/lab/flows/record.mjs

@@ -153,11 +153,12 @@ export function skeletonFor(step        , reads                 )             {
     call of the step at that iteration: imported twice, it is skipped the
     second time, while another run's call -- the same request, sent again,
     answered afresh -- is its own record, since a flow that sends the same
-    words every time is exactly where production's replies differ. One
-    written by hand or uploaded is the same as another that read and sent the
-    same things. */
+    words every time is exactly where production's replies differ. An edited
+    call keeps its run's key, so Import runs skips it too (#587). One written
+    by hand or uploaded is the same as another that read and sent the same
+    things. Mirrored by server.py's record_key. */
 export function recordKey(r                                                           )         {
-  if (r.origin?.kind === "run" && r.origin.run) {
+  if (r.origin?.run) {
     return JSON.stringify({ step: r.step, run: r.origin.run, iteration: r.origin.iteration ?? "" });
   }
   const sorted = (v         )          => Array.isArray(v) ? v.map(sorted)

package/lab/flows/wdl.mjs

@@ -363,6 +363,25 @@ function endOf(s        , from        )         {
   throw new WdlError("an @{ is never closed");
 }
 
+/** Why [template] cannot be evaluated whatever it is evaluated against -- an
+    expression in it that does not parse -- or null. Read before a run, so a
+    template every call would fail on is refused rather than sent. */
+export function templateProblem(template         )                {
+  try { walk(template); return null; }
+  catch (e) { if (e instanceof WdlError) return e.message; throw e; }
+  function walk(v         )       {
+    if (Array.isArray(v)) return v.forEach(walk);
+    if (isObj(v)) return Object.values(v).forEach(walk);
+    if (typeof v !== "string" || v.startsWith("@@")) return;
+    if (v.startsWith("@") && !v.startsWith("@{")) return void parse(v.slice(1));
+    for (let open = v.indexOf("@{"); open >= 0; open = v.indexOf("@{", open)) {
+      const close = endOf(v, open + 2);
+      parse(v.slice(open + 2, close));
+      open = close + 1;
+    }
+  }
+}
+
 /** [template] with every expression in it evaluated: each string that is
     one, or holds one, wherever it sits in objects and arrays. */
 export function evaluate(template         , ctx            )          {

package/lab/kinds/list.mjs

@@ -293,6 +293,8 @@ export const FILTER_SEPARATORS = ["comma", "lines", "semicolon", "none"]
                                                                             
                                         
                    
+                                                                         
+                     
  
 
 /** A base filter type (#334's decision: a registry, not an enum). A reader
@@ -302,10 +304,10 @@ export const FILTER_SEPARATORS = ["comma", "lines", "semicolon", "none"]
                 
                                                                            
                  
-                                                                           
-                                                                           
-                                                                       
-                
+                                                                          
+                                                                            
+                                                                            
+                                
                                              
                                                                             
                                                                     
@@ -340,32 +342,81 @@ function rewriteItem(op                                     )
   return t => t;
 }
 
-/** The function a Rewrite text filter's `op` names: Remove thinking, Remove
-    preamble, or Replace pattern (a regex, global, and its replacement). An
-    unknown op leaves the text as it is; a pattern that will not compile throws,
-    so Ignore failures can carry a run past it. */
-function rewriteText(op                                     )                           {
-  if (op?.kind === "thinking") return removeThinking;
-  if (op?.kind === "preamble") return removePreamble;
-  if (op?.kind === "replace") {
-    const re = new RegExp(String(op.pattern ?? ""), "g");
-    const to = typeof op.replacement === "string" ? op.replacement : "";
-    return text => text.replace(re, to);
+/** The function a Rewrite text filter's `op` names (#557): Remove thinking,
+    Remove preamble, Trim, Replace text (literal, every occurrence), Replace
+    pattern (a regex, global, and its replacement), Strip code fences, Change
+    case, Keep first (N lines or characters), or Extract (a pattern's first
+    match, its first group if it has one). It answers null where nothing is
+    left to keep -- Extract found no match -- which rejects the reply. An
+    unknown op leaves the text as it is; a pattern that will not compile
+    throws, so Ignore failures can carry a run past it. */
+function rewriteText(op                                     )                                  {
+  switch (op?.kind) {
+    case "thinking": return removeThinking;
+    case "preamble": return removePreamble;
+    case "trim": return text => text.trim();
+    case "literal": {
+      const find = String(op.find ?? "");
+      const to = typeof op.replacement === "string" ? op.replacement : "";
+      return text => (find ? text.split(find).join(to) : text);
+    }
+    case "replace": {
+      const re = new RegExp(String(op.pattern ?? ""), "g");
+      const to = typeof op.replacement === "string" ? op.replacement : "";
+      return text => text.replace(re, to);
+    }
+    // A fence's own lines go -- the opening one with its language, and the
+    // closing one -- wherever they sit, so a reply fenced whole or a fence
+    // after a line of preamble both read as what was inside.
+    case "fences": return text => text.replace(FENCE_LINE, "");
+    case "case": { const mode = op.mode          ; return text => applyCase(text, mode); }
+    case "first": {
+      const n = op.n          ;
+      return op.unit === "chars" ? text => text.slice(0, n) : text => text.split("\n").slice(0, n).join("\n");
+    }
+    case "extract": {
+      const re = new RegExp(String(op.pattern ?? ""));
+      return text => { const m = re.exec(text); return m ? (m.length > 1 ? m[1] ?? "" : m[0]) : null; };
+    }
+    default: return text => text;
   }
-  return text => text;
 }
 
+const FENCE_LINE = /^[ \t]*```[^`\n]*(?:\n|$)/gm;
+
+/** Keep first's units, for a Rewrite text: lines, or characters. */
+const TEXT_UNITS = ["lines", "chars"]         ;
+
+/** Why a Rewrite text op's pattern does not compile, as a sentence, or none. */
+const patternProblems = (pattern         , at        )           => {
+  if (typeof pattern !== "string" || !pattern) return [`${at} needs a pattern`];
+  try { new RegExp(pattern); return []; } catch { return [`${at}'s pattern does not compile`]; }
+};
+
 /** The lab's base filter types (#405). Generic code asks this registry by
     `filter.type`; it never names an id. `rewrite-text`/`reject-text` read the
-    raw reply before it is read into items (the Pre-format stage); the rest
-    read the items after (the Post-format stage). */
+    reply's text: Pre-format, or Post-format after a kind that reads no items
+    (#557); the rest read the items, Post-format after a kind that does. */
 export const FILTER_TYPES                                  = {
   "rewrite-text": {
-    label: "Rewrite text", about: "Remove thinking, a preamble or a pattern from the reply's text.", scope: "text", conditions: [],
+    label: "Rewrite text", about: "Trim, replace, unfence, recase, shorten or extract from the reply's text.", scope: "text", conditions: [],
     opProblems: (f, at) => opKindProblems(f.op, at, {
-      thinking: [], preamble: [], replace: ["pattern", "replacement"],
+      thinking: [], preamble: [], trim: [], literal: ["find", "replacement"], replace: ["pattern", "replacement"],
+      fences: [], case: ["mode"], first: ["n", "unit"], extract: ["pattern"],
+    }, op => {
+      if (op.kind === "literal" && (typeof op.find !== "string" || !op.find)) return [`${at} needs text to find`];
+      if (op.kind === "extract") return patternProblems(op.pattern, at);
+      if (op.kind === "case" && !CASE_MODES.includes(op.mode          )) return [`${at}'s case needs a mode`];
+      if (op.kind === "first") {
+        return [...(Number.isInteger(op.n) && (op.n          ) >= 1 ? [] : [`${at} needs a whole number of 1 or more`]),
+                ...(TEXT_UNITS.includes(op.unit         ) ? [] : [`${at} keeps lines or characters`])];
+      }
+      return [];
     }),
-    apply: (filter, reply) => ({ body: rewriteText(filter.op)(reply.body), items: reply.items, dropped: [], reject: null }),
+    apply(filter, reply) {
+      const body = rewriteText(filter.op)(reply.body);
+      return { body: body ?? reply.body, items: reply.items, dropped: [], reject: body == null ? filter.id : null };
+    },
   },
   "reject-text": {
     label: "Reject text", about: "Discard the reply when it is blank or matches a pattern.", scope: "text", conditions: ["blank", "regex"], rejects: true,
@@ -471,19 +522,20 @@ export function applyFilter(filter        , reply             )                {
 export function applyFilterSet(filters                   , reply             , stage                   )                {
   const instruction = reply.instruction ?? "";
   let body = reply.body, items = reply.items;
-  const dropped            = [];
+  const dropped            = [], rewrote           = [];
   let emptied                    ;
   for (const filter of filters) {
     const scope = FILTER_TYPES[filter.type]?.scope;
     if (stage && scope && (scope === "text") !== (stage === "text")) continue;
     const out = applyFilter(filter, { body, items, instruction });
+    if (out.body !== body) rewrote.push(filter.id);
     body = out.body;
     if (items.length && !out.items.length) emptied = filter.id;
     items = out.items;
     dropped.push(...out.dropped);
     if (out.reject != null) return { body, items, dropped, reject: out.reject };
   }
-  return { body, items, dropped, reject: null, ...(emptied ? { emptied } : {}) };
+  return { body, items, dropped, reject: null, ...(emptied ? { emptied } : {}), ...(rewrote.length ? { rewrote } : {}) };
 }
 
 // A newline always ends an item, as it does in the real csv/lines parse
@@ -519,6 +571,8 @@ export function tryFilter(filter        )
       return { test, ok: rejected === !test.expectedOutput.trim(), got: rejected ? "rejected" : (scope === "text" ? out.body : out.items.join(", ")) };
     }
     if (scope === "text") {
+      // A Rewrite text that rejects (Extract, matching nothing) leaves nothing.
+      if (out.reject != null) return { test, ok: !test.expectedOutput.trim(), got: "rejected" };
       const got = out.body.trim();
       return { test, ok: got === test.expectedOutput.trim(), got };
     }

package/lab/metrics/builtin.mjs

@@ -139,9 +139,9 @@ function verdictOf(said        )
     can hold one. A reply that still cannot be read is asked for once more;
     a second, or a grader that failed, is the grader's error -- said as one,
     so it is not read as the reply failing. [ok] says whether a verdict read
-    is one this metric can use. */
+    is one this metric can use: true, or why not. */
 async function graded(ctx           , prompt        , schema                         ,
-                      ok                                               = () => true)                                        {
+                      ok                                                        = () => true)                                        {
   if (!ctx.ask) throw new Error("a model-graded metric needs a grader, and this run has none");
   let last = "";
   for (let attempt = 0; attempt < 2; attempt++) {
@@ -149,14 +149,21 @@ async function graded(ctx           , prompt        , schema
     try { said = await ctx.ask(prompt, schema); } catch (e) { throw new Error(`grader error: ${e instanceof Error ? e.message : String(e)}`); }
     try {
       const v = verdictOf(said);
-      if (ok(v)) return v;
-      last = `the grader said ${short(JSON.stringify(v))}`;
+      const usable = ok(v);
+      if (usable === true) return v;
+      last = typeof usable === "string" ? usable : `the grader said ${short(JSON.stringify(v))}`;
     } catch (e) { last = e instanceof Error ? e.message : String(e); }
   }
   throw new Error(`grader error: ${last}`);
 }
+/** Whether a pass-or-fail verdict's score is on the scale it was asked for.
+    One off it -- a 4 echoing a 1-to-5 rating -- is not clamped to a score the
+    grader never gave: it is read as unreadable, asked for again, then the
+    grader's error (#514). */
+const onScale = (v                              )                =>
+  typeof v.score !== "number" || (v.score >= 0 && v.score <= 1) || `the grader scored ${v.score}, off the 0 to 1 scale`;
 const gradedPass = (v                              , threshold        )                => {
-  const score = typeof v.score === "number" ? Math.max(0, Math.min(1, v.score)) : v.pass === true ? 1 : 0;
+  const score = typeof v.score === "number" ? v.score : v.pass === true ? 1 : 0;
   const pass = typeof v.pass === "boolean" ? v.pass && score >= threshold : score >= threshold;
   return { pass, score, reason: str(v.reason) || (pass ? "the grader passed it" : "the grader failed it") };
 };
@@ -339,6 +346,7 @@ const metrics                              = {
   // rules discarded is the finding.
   discarded: {
     family: RESULT,
+    readsResult: true,
     label: "Discarded",
     description: "Passes only when the job's rules discarded the reply.",
     options: [],
@@ -348,8 +356,26 @@ const metrics                              = {
       return ok(thrown, thrown ? "discarded" : input.error ? input.error : `kept ${input.terms.length} items`);
     },
   },
+  // A case that expects no answer at all: the connection answering an error
+  // -- a 400 for a file it can't read -- is the finding. A discard is
+  // Discarded's, at whichever stage it was.
+  refused: {
+    family: RESULT,
+    readsResult: true,
+    readsRefusal: true,
+    label: "Refused",
+    description: "Passes only when the target's call was refused: an error from the connection, not a reply.",
+    options: [],
+    defaults: () => ({}),
+    score: (input) => {
+      if (!input.error) return ok(false, "replied");
+      const refused = !/^(stage \d+: )?discarded: /.test(input.error);
+      return ok(refused, refused ? `refused: ${input.error}` : input.error);
+    },
+  },
   "item-count": {
     family: RESULT,
+    readsResult: true,
     label: "Item count",
     description: "Passes when the number of items is within the bounds.",
     options: [{ key: "min", label: "At least", type: "number" }, { key: "max", label: "At most", type: "number" }],
@@ -385,6 +411,7 @@ const metrics                              = {
   },
   latency: {
     family: RESULT,
+    readsResult: true,
     label: "Latency",
     description: "Passes when the reply came back within the time set.",
     options: [{ key: "max", label: "At most (ms)", type: "number" }],
@@ -472,7 +499,7 @@ const metrics                              = {
     validate: (m, at, bad) => needs(m, ["rubric"], at, bad),
     score: async (input, m, ctx) => gradedPass(await graded(ctx,
       `You are grading an output against a rubric.\n\n${taskOf(input, "")}Rubric:\n${str(m.rubric)}\n\nOutput:\n${input.text}\n\n${REPLY}`,
-      VERDICT_SCHEMA), num(m.threshold) ?? 0.5),
+      VERDICT_SCHEMA, onScale), num(m.threshold) ?? 0.5),
   },
   factuality: {
     family: GRADED,
@@ -488,7 +515,7 @@ const metrics                              = {
     score: async (input, m, ctx) => gradedPass(await graded(ctx,
       `You are checking an output for factual consistency with a reference. Differences in wording or detail are `
       + `fine; a contradiction, or a fact the reference does not support, is not.\n\n${taskOf(input, "")}Reference:\n`
-      + `${str(m.reference).trim() || recorded(input)}\n\nOutput:\n${input.text}\n\n${REPLY}`, VERDICT_SCHEMA), num(m.threshold) ?? 0.5),
+      + `${str(m.reference).trim() || recorded(input)}\n\nOutput:\n${input.text}\n\n${REPLY}`, VERDICT_SCHEMA, onScale), num(m.threshold) ?? 0.5),
   },
   "judge-vs-production": {
     family: GRADED,

package/lab/run-evals.js

@@ -479,20 +479,24 @@ function prepare(bin, file, c) {
 
 // ---- The type → handler registry ------------------------------------------
 // Design §4. What a file *is* to a run is decided here and only here, so the
-// queue and the page never learn about file types, and pdf and xlsx are
-// later entries that touch nothing but this map.
+// queue and the page never learn about file types, and xlsx is a later
+// entry that touches nothing but this map.
 //
 // A file with no mapper is passed through anyway -- decoded as UTF-8 with
 // replacement and joined at {text} like the text handler, and flagged in the
-// transcript. Stating the failure mode beats hiding it: a PDF's text layer
-// is not extracted, so its streams arrive as arbitrary bytes, and a run
-// that read them "well" would be a finding about nothing.
+// transcript. Stating the failure mode beats hiding it: a binary format's
+// bytes arrive as mojibake, and a run that read them "well" would be a
+// finding about nothing.
 const HANDLERS = new Map();
 [
   // Images: prepared to the profile's pixel budget by ImageMagick and sent
   // as the image content part. The formats are server.py's FILE_TYPES
   // image half, so nothing accepted for upload has no executor to run it.
   ["image", "jpg", "jpeg", "png", "heic", "heif", "dng", "tif", "tiff"],
+  // Documents: the file's bytes, unprocessed, handed to an API that reads
+  // them whole (core.DOCUMENT_TYPES); a connection that cannot is refused
+  // before the run, by validatePipeline.
+  ["document", ...Object.keys(core.DOCUMENT_TYPES).map(e => e.slice(1))],
   // Text-ish: decoded UTF-8, joining the stage at {text} -- the text-content
   // semantics the canvas already has (core.textPrompt).
   ["text", "txt", "md", "csv"],
@@ -511,8 +515,8 @@ const UNMAPPED = "the file has no handler, so it was passed through as raw "
 
 /**
  * One file of the Source made into what the job consumes: an image as the
- * dataUrl at [conn]'s budget, or text at {text}. A file with no mapper
- * still runs, and says so.
+ * dataUrl at [conn]'s budget, a document as its bytes, or text at {text}. A
+ * file with no mapper still runs, and says so.
  */
 function itemContent(bin, file, conn) {
   const kind = HANDLERS.get(extOf(file));
@@ -520,6 +524,7 @@ function itemContent(bin, file, conn) {
     const { dataUrl, dims } = prepare(bin, file, conn);
     return { kind, dataUrl, text: null, flags: [] };
   }
+  if (kind === "document") return { kind, dataUrl: documentOf(file), text: null, flags: [] };
   // TextDecoder with fatal unset replaces malformed bytes with U+FFFD --
   // what "decoded as UTF-8 with replacement" means, for a mapped-text file
   // and an unmapped one alike.
@@ -527,6 +532,13 @@ function itemContent(bin, file, conn) {
   return { kind: kind ?? "unmapped", dataUrl: null, text, flags: kind ? [] : [UNMAPPED] };
 }
 
+/** A document file as the attachment a stage sends: its bytes, its media
+    type and its name. */
+function documentOf(file) {
+  const name = path.basename(file);
+  return core.documentUrl(name, core.documentTypeOf(name).media, fs.readFileSync(file).toString("base64"));
+}
+
 // ---- Reaching a model ----------------------------------------------------
 
 /**
@@ -712,7 +724,9 @@ async function ask(to, c, prompt, dataUrl, schema = null) {
     if (!r.ok) return said({ error: j.error?.message || j.error || `HTTP ${r.status}` });
     const { raw, finishReason } = core.connectionReply(c, j);
     const error = core.connectionReplyProblem(c, raw, finishReason);
-    return said(error ? { raw, error } : { raw });
+    // Why it stopped, and the limit sent where the limit stopped it (#597).
+    const finish = core.replyFinish(c, req.body, finishReason);
+    return said(error ? { raw, error, ...finish } : { raw, ...finish });
   } catch (e) {
     if (e?.name === "TimeoutError" || e?.name === "AbortError") {
       return said({ error: `the model gave no reply within ${seconds}s — timed out` });
@@ -737,9 +751,10 @@ async function ask(to, c, prompt, dataUrl, schema = null) {
 // this invocation did, so resume and re-run item agree with a fresh run.
 
 // What a file *is* to a run, the same registry §4 gives the graded set: an
-// image is prepared to the profile's pixel budget; a text file is decoded
-// UTF-8 and joined at {text}; anything else is passed through as UTF-8 with
-// replacement and flagged, so the failure mode is stated rather than hidden.
+// image is prepared to the profile's pixel budget; a document is sent whole;
+// a text file is decoded UTF-8 and joined at {text}; anything else is passed
+// through as UTF-8 with replacement and flagged, so the failure mode is
+// stated rather than hidden.
 const RUN_IMAGE_TYPES = new Set([".jpg", ".jpeg", ".png", ".heic", ".heif", ".dng", ".tif", ".tiff"]);
 const RUN_TEXT_TYPES = new Set([".txt", ".md", ".csv"]);
 
@@ -916,6 +931,7 @@ async function runSnapshot(o) {
     : files.map(name => {
         const ext = path.extname(name).toLowerCase();
         if (RUN_IMAGE_TYPES.has(ext)) return { name, kind: "image" };
+        if (core.documentTypeOf(name)) return { name, kind: "document" };
         // A record (flows/record.ts): one call of a Power Automate step, what
         // an HTTP Request is rebuilt from.
         if (ext === ".json") return { name, kind: "record" };
@@ -981,6 +997,11 @@ async function runSnapshot(o) {
           break;
         }
       }
+      if (item.kind === "document") {
+        const file = path.join(o.source, item.name);
+        if (!fs.existsSync(file)) { failed = `the file ${item.name} is not in the source`; break; }
+        dataUrl = documentOf(file);
+      }
       let record = null;
       if (item.kind === "record") {
         const file = path.join(o.source, item.name);
@@ -1450,7 +1471,7 @@ async function main() {
       found: s.found, missed: s.missed, invented: s.invented, count: s.count,
       metrics: s.metrics, watch: s.watch,
       terms: res.terms || [], ms: res.ms,
-      // What a file type was to this run -- "image", "text", "unmapped".
+      // What a file type was to this run -- "image", "document", "text", "unmapped".
       type: item?.kind ?? null,
       // What the model actually said. #248 wants a failure written out as a
       // task an agent can act on, and "missed water" without the reply that