evals-lab 0.11.0 → 0.13.0

package/lab/kinds/list.mjs

@@ -293,6 +293,8 @@ export const FILTER_SEPARATORS = ["comma", "lines", "semicolon", "none"]
                                                                             
                                         
                    
+                                                                         
+                     
  
 
 /** A base filter type (#334's decision: a registry, not an enum). A reader
@@ -471,19 +473,20 @@ export function applyFilter(filter        , reply             )                {
 export function applyFilterSet(filters                   , reply             , stage                   )                {
   const instruction = reply.instruction ?? "";
   let body = reply.body, items = reply.items;
-  const dropped            = [];
+  const dropped            = [], rewrote           = [];
   let emptied                    ;
   for (const filter of filters) {
     const scope = FILTER_TYPES[filter.type]?.scope;
     if (stage && scope && (scope === "text") !== (stage === "text")) continue;
     const out = applyFilter(filter, { body, items, instruction });
+    if (out.body !== body) rewrote.push(filter.id);
     body = out.body;
     if (items.length && !out.items.length) emptied = filter.id;
     items = out.items;
     dropped.push(...out.dropped);
     if (out.reject != null) return { body, items, dropped, reject: out.reject };
   }
-  return { body, items, dropped, reject: null, ...(emptied ? { emptied } : {}) };
+  return { body, items, dropped, reject: null, ...(emptied ? { emptied } : {}), ...(rewrote.length ? { rewrote } : {}) };
 }
 
 // A newline always ends an item, as it does in the real csv/lines parse

package/lab/metrics/builtin.mjs

@@ -139,9 +139,9 @@ function verdictOf(said        )
     can hold one. A reply that still cannot be read is asked for once more;
     a second, or a grader that failed, is the grader's error -- said as one,
     so it is not read as the reply failing. [ok] says whether a verdict read
-    is one this metric can use. */
+    is one this metric can use: true, or why not. */
 async function graded(ctx           , prompt        , schema                         ,
-                      ok                                               = () => true)                                        {
+                      ok                                                        = () => true)                                        {
   if (!ctx.ask) throw new Error("a model-graded metric needs a grader, and this run has none");
   let last = "";
   for (let attempt = 0; attempt < 2; attempt++) {
@@ -149,14 +149,21 @@ async function graded(ctx           , prompt        , schema
     try { said = await ctx.ask(prompt, schema); } catch (e) { throw new Error(`grader error: ${e instanceof Error ? e.message : String(e)}`); }
     try {
       const v = verdictOf(said);
-      if (ok(v)) return v;
-      last = `the grader said ${short(JSON.stringify(v))}`;
+      const usable = ok(v);
+      if (usable === true) return v;
+      last = typeof usable === "string" ? usable : `the grader said ${short(JSON.stringify(v))}`;
     } catch (e) { last = e instanceof Error ? e.message : String(e); }
   }
   throw new Error(`grader error: ${last}`);
 }
+/** Whether a pass-or-fail verdict's score is on the scale it was asked for.
+    One off it -- a 4 echoing a 1-to-5 rating -- is not clamped to a score the
+    grader never gave: it is read as unreadable, asked for again, then the
+    grader's error (#514). */
+const onScale = (v                              )                =>
+  typeof v.score !== "number" || (v.score >= 0 && v.score <= 1) || `the grader scored ${v.score}, off the 0 to 1 scale`;
 const gradedPass = (v                              , threshold        )                => {
-  const score = typeof v.score === "number" ? Math.max(0, Math.min(1, v.score)) : v.pass === true ? 1 : 0;
+  const score = typeof v.score === "number" ? v.score : v.pass === true ? 1 : 0;
   const pass = typeof v.pass === "boolean" ? v.pass && score >= threshold : score >= threshold;
   return { pass, score, reason: str(v.reason) || (pass ? "the grader passed it" : "the grader failed it") };
 };
@@ -339,6 +346,7 @@ const metrics                              = {
   // rules discarded is the finding.
   discarded: {
     family: RESULT,
+    readsResult: true,
     label: "Discarded",
     description: "Passes only when the job's rules discarded the reply.",
     options: [],
@@ -348,8 +356,26 @@ const metrics                              = {
       return ok(thrown, thrown ? "discarded" : input.error ? input.error : `kept ${input.terms.length} items`);
     },
   },
+  // A case that expects no answer at all: the connection answering an error
+  // -- a 400 for a file it can't read -- is the finding. A discard is
+  // Discarded's, at whichever stage it was.
+  refused: {
+    family: RESULT,
+    readsResult: true,
+    readsRefusal: true,
+    label: "Refused",
+    description: "Passes only when the target's call was refused: an error from the connection, not a reply.",
+    options: [],
+    defaults: () => ({}),
+    score: (input) => {
+      if (!input.error) return ok(false, "replied");
+      const refused = !/^(stage \d+: )?discarded: /.test(input.error);
+      return ok(refused, refused ? `refused: ${input.error}` : input.error);
+    },
+  },
   "item-count": {
     family: RESULT,
+    readsResult: true,
     label: "Item count",
     description: "Passes when the number of items is within the bounds.",
     options: [{ key: "min", label: "At least", type: "number" }, { key: "max", label: "At most", type: "number" }],
@@ -385,6 +411,7 @@ const metrics                              = {
   },
   latency: {
     family: RESULT,
+    readsResult: true,
     label: "Latency",
     description: "Passes when the reply came back within the time set.",
     options: [{ key: "max", label: "At most (ms)", type: "number" }],
@@ -472,7 +499,7 @@ const metrics                              = {
     validate: (m, at, bad) => needs(m, ["rubric"], at, bad),
     score: async (input, m, ctx) => gradedPass(await graded(ctx,
       `You are grading an output against a rubric.\n\n${taskOf(input, "")}Rubric:\n${str(m.rubric)}\n\nOutput:\n${input.text}\n\n${REPLY}`,
-      VERDICT_SCHEMA), num(m.threshold) ?? 0.5),
+      VERDICT_SCHEMA, onScale), num(m.threshold) ?? 0.5),
   },
   factuality: {
     family: GRADED,
@@ -488,7 +515,7 @@ const metrics                              = {
     score: async (input, m, ctx) => gradedPass(await graded(ctx,
       `You are checking an output for factual consistency with a reference. Differences in wording or detail are `
       + `fine; a contradiction, or a fact the reference does not support, is not.\n\n${taskOf(input, "")}Reference:\n`
-      + `${str(m.reference).trim() || recorded(input)}\n\nOutput:\n${input.text}\n\n${REPLY}`, VERDICT_SCHEMA), num(m.threshold) ?? 0.5),
+      + `${str(m.reference).trim() || recorded(input)}\n\nOutput:\n${input.text}\n\n${REPLY}`, VERDICT_SCHEMA, onScale), num(m.threshold) ?? 0.5),
   },
   "judge-vs-production": {
     family: GRADED,

package/lab/run-evals.js

@@ -479,20 +479,24 @@ function prepare(bin, file, c) {
 
 // ---- The type → handler registry ------------------------------------------
 // Design §4. What a file *is* to a run is decided here and only here, so the
-// queue and the page never learn about file types, and pdf and xlsx are
-// later entries that touch nothing but this map.
+// queue and the page never learn about file types, and xlsx is a later
+// entry that touches nothing but this map.
 //
 // A file with no mapper is passed through anyway -- decoded as UTF-8 with
 // replacement and joined at {text} like the text handler, and flagged in the
-// transcript. Stating the failure mode beats hiding it: a PDF's text layer
-// is not extracted, so its streams arrive as arbitrary bytes, and a run
-// that read them "well" would be a finding about nothing.
+// transcript. Stating the failure mode beats hiding it: a binary format's
+// bytes arrive as mojibake, and a run that read them "well" would be a
+// finding about nothing.
 const HANDLERS = new Map();
 [
   // Images: prepared to the profile's pixel budget by ImageMagick and sent
   // as the image content part. The formats are server.py's FILE_TYPES
   // image half, so nothing accepted for upload has no executor to run it.
   ["image", "jpg", "jpeg", "png", "heic", "heif", "dng", "tif", "tiff"],
+  // Documents: the file's bytes, unprocessed, handed to an API that reads
+  // them whole (core.DOCUMENT_TYPES); a connection that cannot is refused
+  // before the run, by validatePipeline.
+  ["document", ...Object.keys(core.DOCUMENT_TYPES).map(e => e.slice(1))],
   // Text-ish: decoded UTF-8, joining the stage at {text} -- the text-content
   // semantics the canvas already has (core.textPrompt).
   ["text", "txt", "md", "csv"],
@@ -511,8 +515,8 @@ const UNMAPPED = "the file has no handler, so it was passed through as raw "
 
 /**
  * One file of the Source made into what the job consumes: an image as the
- * dataUrl at [conn]'s budget, or text at {text}. A file with no mapper
- * still runs, and says so.
+ * dataUrl at [conn]'s budget, a document as its bytes, or text at {text}. A
+ * file with no mapper still runs, and says so.
  */
 function itemContent(bin, file, conn) {
   const kind = HANDLERS.get(extOf(file));
@@ -520,6 +524,7 @@ function itemContent(bin, file, conn) {
     const { dataUrl, dims } = prepare(bin, file, conn);
     return { kind, dataUrl, text: null, flags: [] };
   }
+  if (kind === "document") return { kind, dataUrl: documentOf(file), text: null, flags: [] };
   // TextDecoder with fatal unset replaces malformed bytes with U+FFFD --
   // what "decoded as UTF-8 with replacement" means, for a mapped-text file
   // and an unmapped one alike.
@@ -527,6 +532,13 @@ function itemContent(bin, file, conn) {
   return { kind: kind ?? "unmapped", dataUrl: null, text, flags: kind ? [] : [UNMAPPED] };
 }
 
+/** A document file as the attachment a stage sends: its bytes, its media
+    type and its name. */
+function documentOf(file) {
+  const name = path.basename(file);
+  return core.documentUrl(name, core.documentTypeOf(name).media, fs.readFileSync(file).toString("base64"));
+}
+
 // ---- Reaching a model ----------------------------------------------------
 
 /**
@@ -737,9 +749,10 @@ async function ask(to, c, prompt, dataUrl, schema = null) {
 // this invocation did, so resume and re-run item agree with a fresh run.
 
 // What a file *is* to a run, the same registry §4 gives the graded set: an
-// image is prepared to the profile's pixel budget; a text file is decoded
-// UTF-8 and joined at {text}; anything else is passed through as UTF-8 with
-// replacement and flagged, so the failure mode is stated rather than hidden.
+// image is prepared to the profile's pixel budget; a document is sent whole;
+// a text file is decoded UTF-8 and joined at {text}; anything else is passed
+// through as UTF-8 with replacement and flagged, so the failure mode is
+// stated rather than hidden.
 const RUN_IMAGE_TYPES = new Set([".jpg", ".jpeg", ".png", ".heic", ".heif", ".dng", ".tif", ".tiff"]);
 const RUN_TEXT_TYPES = new Set([".txt", ".md", ".csv"]);
 
@@ -916,6 +929,7 @@ async function runSnapshot(o) {
     : files.map(name => {
         const ext = path.extname(name).toLowerCase();
         if (RUN_IMAGE_TYPES.has(ext)) return { name, kind: "image" };
+        if (core.documentTypeOf(name)) return { name, kind: "document" };
         // A record (flows/record.ts): one call of a Power Automate step, what
         // an HTTP Request is rebuilt from.
         if (ext === ".json") return { name, kind: "record" };
@@ -981,6 +995,11 @@ async function runSnapshot(o) {
           break;
         }
       }
+      if (item.kind === "document") {
+        const file = path.join(o.source, item.name);
+        if (!fs.existsSync(file)) { failed = `the file ${item.name} is not in the source`; break; }
+        dataUrl = documentOf(file);
+      }
       let record = null;
       if (item.kind === "record") {
         const file = path.join(o.source, item.name);
@@ -1450,7 +1469,7 @@ async function main() {
       found: s.found, missed: s.missed, invented: s.invented, count: s.count,
       metrics: s.metrics, watch: s.watch,
       terms: res.terms || [], ms: res.ms,
-      // What a file type was to this run -- "image", "text", "unmapped".
+      // What a file type was to this run -- "image", "document", "text", "unmapped".
       type: item?.kind ?? null,
       // What the model actually said. #248 wants a failure written out as a
       // task an agent can act on, and "missed water" without the reply that