evals-lab 0.12.0 → 0.13.0

package/CHANGELOG.md

@@ -5,6 +5,55 @@ Newest first. Read a release's **Upgrade notes** before installing it: an
 upgrade can rewrite what the lab keeps in your data directory, and an older
 version cannot always read it back.
 
+## 0.13.0
+
+### Upgrade notes
+
+- Pipelines and datasets keep their versions (17 and 9), so nothing in your
+  data directory is rewritten. A pipeline that uses the new Refused metric
+  needs 0.13.0 to grade it.
+
+### Added
+
+- A Refused metric, for a case that expects the target's call to be refused.
+- A PDF item goes to the model whole, as a document; a connection that can't
+  read one is refused before the run.
+- AI Connections and Connection Profiles select several rows to remove at
+  once, with one Undo; Connection Profiles show their connection.
+- Copy or move… on an eval group's, a filter set's and a Source's own page.
+
+### Changed
+
+- One date format everywhere, and Home lists recent runs.
+- Dropdowns that name things read A–Z; Model, Environment and Flow open the
+  Switcher.
+- Empty values read as a short parenthetical, such as (No grader), and
+  explanations are shorter.
+- A run's jobs sit in one Jobs group, with Add job inside it.
+- A Supplied Source takes several files in one Upload.
+- Enter in a dialog's field presses its main button; in Run settings it
+  leaves the field.
+- Stop reads Stopping… until the run ends, and History can stop a run.
+
+### Fixed
+
+- An errored result fails on its error and is not graded.
+- Metrics and the grader read the reply after its Pre-format filters, and
+  Results shows that reply.
+- A linked group whose grader is gone can't run, and Results says why a
+  graded metric never ran; an off-scale grader score is the grader's error.
+- An Edited target whose expression doesn't parse is refused before the run.
+- Pin names the newest version the server holds.
+- Demo targets answer on a connection the workspace can use.
+- Loop repeats a live capture, and the Comment stays with its pipeline.
+- A Re-sync that needed a Microsoft sign-in runs once the page is back.
+- A dynamic Source's fetch names the lab as its User-Agent.
+- Emptying the trash leaves a connection shared only with the purged
+  workspace shared with none, not All; a lab import's rollback leaves
+  another import's connection intact.
+- A stored Echo connection is no longer listed beside the built-in Echo
+  target.
+
 ## 0.12.0
 
 ### Upgrade notes

package/lab/VERSION

@@ -1 +1 @@
-0.12.0 (2026.10.12-718)
+0.13.0 (2026.10.13-794)

package/lab/evals-core.mjs

@@ -34,7 +34,7 @@
 
 import yaml from "./js-yaml.mjs";
                                                                                                                                                  
-import { evaluate, asText, WdlError,                 } from "./flows/wdl.mjs";
+import { evaluate, asText, templateProblem, WdlError,                 } from "./flows/wdl.mjs";
 import { stepsOf as flowStepsOf, actionsByName, readerFor, readsOf,                          } from "./flows/record.mjs";
 import { flowApi,              } from "./flows/flowApi.mjs";
 import registerMetrics from "./metrics/builtin.mjs";
@@ -607,6 +607,9 @@ export const FILTER_STAGE_LABEL                              = { text: "Pre-form
                                                                             
                                               
                 
+                                                                            
+                                                                         
+                                           
  
 
 /** How much of one text a run keeps -- a reply's body, a grader's prompt or
@@ -897,6 +900,14 @@ export function kept(text        , max = KEPT_CHARS)         {
                     
                                                                                     
                        
+                                                                              
+                                                                               
+                                                                          
+                        
+                                                                          
+                                                                        
+                       
+                         
                                                                              
                                                                            
                  
@@ -1690,9 +1701,17 @@ function datasetRules(body         )                  {
                                                                                                   
                                     
                           
+                                                                             
+                                                                             
+                                                            
+                      
                                                                            
                                  
                     
+                                                                             
+                                                                             
+                                                                               
+                                        
                                                                           
                                                                                
                                                                          
@@ -1993,6 +2012,35 @@ function asNumber(v         )                {
 // a tuned local engine is a llama.cpp profile with its settings.
 const CONNECTION_TYPES                                 = Object.create(null);
 
+// ---- documents (#558) ------------------------------------------------------
+// A file a model's API reads whole, sent unprocessed: the lab does not parse,
+// render or extract it. It travels where an image does -- the item's
+// attachment, a data URL -- carrying its name, which some APIs want beside
+// the bytes. Only a connection type marked `documents` is handed one.
+const DOCUMENT_TYPES                                                   = {
+  ".pdf": { media: "application/pdf", label: "PDF" },
+};
+
+/** The DOCUMENT_TYPES entry a file's name is, or null. */
+function documentTypeOf(name        )                                          {
+  const dot = String(name).lastIndexOf(".");
+  return dot < 0 ? null : DOCUMENT_TYPES[String(name).slice(dot).toLowerCase()] ?? null;
+}
+
+/** A document's bytes, base64, as the attachment a stage sends. */
+function documentUrl(name        , media        , base64        )         {
+  return `data:${media};name=${encodeURIComponent(name)};base64,${base64}`;
+}
+
+/** An attachment's media type, bytes and name (null for an image, which
+    carries none), or null for anything that is not a base64 data URL. */
+function attachmentOf(dataUrl        )                                                              {
+  const m = /^data:([^;,]+)((?:;[^;,=]+=[^;,]*)*);base64,(.*)$/s.exec(dataUrl);
+  if (!m) return null;
+  const name = /;name=([^;,]*)/.exec(m[2] )?.[1];
+  return { media: m[1] , data: m[3] , name: name == null ? null : decodeURIComponent(name) };
+}
+
 /** The key [key] as the header most endpoints expect, when there is one. */
 const bearer = (key                           )                         => (key ? { Authorization: "Bearer " + key } : {});
 
@@ -2105,6 +2153,13 @@ CONNECTION_TYPES.recorded = {
   request: sendsNothing, parseReply: sendsNothing, listModels: sendsNothing, parseModels: () => [],
 };
 
+/** Whether Setup and the connection pickers list a stored connection. A local
+    type's (Echo, Recorded) is a target step's stand-in now (`localSteps`), so
+    a row of one is kept only for the pipelines and runs that name it. */
+function listedConnection(conn                           )          {
+  return !CONNECTION_TYPES[typeOf(conn                               )]?.local;
+}
+
 /** A local type's answer to one stage, as a model's would come back. An item
     with no text of its own (Prompt only) is answered from the prompt, and
     that reply is read against nothing -- the prompt is the reply, not an
@@ -2120,9 +2175,19 @@ function localAnswer(conn          , text               , sent        , record
 const chatSchema = (body                         , schema                         )                          =>
   ({ ...body, response_format: { type: "json_schema", json_schema: { name: "reply", strict: true, schema } } });
 
+/** An item's attachment as a chat completions content part: an image as
+    image_url, a document as a file part, named, its bytes a data URL. */
+function chatAttachment(dataUrl        )         {
+  const a = attachmentOf(dataUrl);
+  if (!a || a.media.startsWith("image/")) return { type: "image_url", image_url: { url: dataUrl } };
+  return { type: "file", file: { filename: a.name ?? "document", file_data: `data:${a.media};base64,${a.data}` } };
+}
+
 CONNECTION_TYPES["openai-compatible"] = {
   id: "openai-compatible", label: "OpenAI-compatible",
   description: "OpenAI, OpenRouter, vLLM, LM Studio: any chat completions endpoint.",
+  // A hosted provider wants a key; a server of your own may not.
+  keyOptional: (conn) => !isHostedUrl(String(conn.url ?? "")),
   settings: [
     // A reasoning model refuses a temperature.
     { key: "temperature", label: "Temperature", control: "text", default: "", hint: "",
@@ -2145,11 +2210,14 @@ CONNECTION_TYPES["openai-compatible"] = {
         ...(predict == null ? {} : hosted ? { max_completion_tokens: predict } : { max_tokens: predict }),
         messages: [{ role: "user", content: [
           { type: "text", text: prompt },
-          ...(dataUrl ? [{ type: "image_url", image_url: { url: dataUrl } }] : []),
+          ...(dataUrl ? [chatAttachment(dataUrl)] : []),
         ] }],
       },
     };
   },
+  // OpenAI's file part; a compatible server that has none refuses the call,
+  // and the refusal is that item's error.
+  documents: true,
   structured: chatSchema,
   parseReply: chatReply,
   listModels, parseModels: idsFrom,
@@ -2174,15 +2242,15 @@ CONNECTION_TYPES.anthropic = {
   answers: ["prompt", "request"],
   speaks: () => "anthropic",
   // The Messages API: the x-api-key and anthropic-version headers, base64
-  // image content blocks, and /v1/models for the list.
+  // image and document content blocks, and /v1/models for the list.
+  documents: true,
   request(conn, prompt, dataUrl, base, key) {
     const temp = asNumber(conn.temperature);
     const predict = asNumber(conn.maxTokens) ?? ANTHROPIC_MAX_TOKENS;
     const content           = [];
-    if (dataUrl) {
-      const m = /^data:(image\/[^;]+);base64,(.*)$/s.exec(dataUrl);
-      if (m) content.push({ type: "image", source: { type: "base64", media_type: m[1], data: m[2] } });
-    }
+    const a = dataUrl ? attachmentOf(dataUrl) : null;
+    if (a?.media.startsWith("image/")) content.push({ type: "image", source: { type: "base64", media_type: a.media, data: a.data } });
+    else if (a) content.push({ type: "document", source: { type: "base64", media_type: a.media, data: a.data } });
     content.push({ type: "text", text: prompt });
     return {
       url: base + "/messages",
@@ -2227,6 +2295,7 @@ const SPEED_IMAGE = "before", SPEED_CACHE = "off", SPEED_THINKING = "off";
 CONNECTION_TYPES["llama.cpp"] = {
   id: "llama.cpp", label: "llama.cpp",
   description: "A llama-server you run, with its sampler settings.",
+  keyOptional: () => true,
   settings: [
     { key: "temperature", label: "Temperature", control: "text", default: "", hint: "" },
     { key: "seed", label: "Seed", control: "text", default: "", hint: "" },
@@ -2511,6 +2580,7 @@ CONNECTION_TYPES.http = {
                 ...Object.entries(HTTP_APIS).map(([value, a]) => ({ value, label: a.label }))] },
   ],
   answers: ["request"],
+  keyOptional: (conn) => conn.auth === "none",
   speaks(conn) {
     const api = String(conn.requestFormat || "") || httpApiFor(originBase(conn.url));
     return HTTP_APIS[api] && api !== "raw" ? api : null;
@@ -2797,7 +2867,7 @@ function readCase(kase      , res                                   , plain = fa
     }
     return r ? [r                 ] : [];
   });
-  const s = scoreOf(metrics) ?? { pass: true, score: null, metrics };
+  const s = erroredScore(scoreOf(metrics), res.error) ?? { pass: !res.error, score: null, metrics };
   const counted = metrics.filter(r => r.weight !== 0);
   return { ...s, metrics, found: s.found ?? [], missed: s.missed ?? [], invented: s.invented ?? [],
            discarded: !!res.error, count: res.error ? 0 : (res.terms || []).length,
@@ -3195,6 +3265,7 @@ async function runPipeline(stages                  , dataUrl               , cal
     const pre = applyFilterSet(st.before || [], { body: res.raw ?? "", items: [], instruction: against }, "text");
     const by = (filter        , stage             , rejected         )             => ({ filter, stage, job: i + 1, rejected });
     if (pre.reject != null) return result({ error: named(`discarded: ${pre.reject}`), filteredBy: by(pre.reject, "text", true) });
+    if (pre.rewrote) transcript[transcript.length - 1] .filtered = { got: pre.body, by: pre.rewrote };
     const read = reader.read(pre.body, { echo: against, parsing: opts.parsing, settings: st.settings });
     if (read.error) return result({ error: named(read.error) });
     const mod = applyModifiers(mods[i], kind, read.value, { instruction: against, body: read.body ?? res.raw ?? "" });
@@ -3731,16 +3802,17 @@ function onlyFields(obj        , at        , allowed                   , bad
 
 // A Source is referenced, with the two numbers that decide how many items a
 // run over it is: the first N files, and how many times over.
-// A folder of files: images sent as images, text files as {text}. The
-// extensions are server.py's FILE_TYPES, which the server holds uploads to.
+// A folder of files: images sent as images, documents whole (DOCUMENT_TYPES),
+// text files as {text}. The extensions are server.py's FILE_TYPES, which the
+// server holds uploads to.
 const FILE_LIBRARY_TEXT = [".txt", ".md", ".csv"]         ;
 SOURCE_TYPES.files = {
   label: "File Library",
   icon: "folder",
-  description: "A folder of images and text files you upload.",
+  description: "A folder of images, PDFs and text files you upload.",
   group: "Files",
   noun: "file",
-  uploads: [".jpg", ".jpeg", ".png", ".heic", ".heif", ".dng", ".tif", ".tiff", ...FILE_LIBRARY_TEXT],
+  uploads: [".jpg", ".jpeg", ".png", ".heic", ".heif", ".dng", ".tif", ".tiff", ...Object.keys(DOCUMENT_TYPES), ...FILE_LIBRARY_TEXT],
   sendsImage: true,
   hasActions: false,
   itemText: name => FILE_LIBRARY_TEXT.some(ext => name.toLowerCase().endsWith(ext)),
@@ -3757,7 +3829,9 @@ SOURCE_TYPES.workflow = {
   icon: "workflow",
   description: "A cloud flow, with records of its steps' calls from its run history.",
   group: "Workflows",
-  noun: "record",
+  // Each item reads as a call, as the Source's Calls group names them, though
+  // the file holding one is a record.
+  noun: "call",
   uploads: [".json"],
   sendsImage: false,
   hasActions: true,
@@ -3846,7 +3920,7 @@ const API_METHODS = ["GET", "POST", "PUT", "PATCH", "DELETE"];
 DYNAMIC_PROVIDERS["api"] = {      // provider: the API call adapter's own id
   id: "api",                      // provider: its id, the row's config.provider
   label: "API call",
-  icon: "square-code",
+  icon: "arrow-left-right",
   noun: "call",
   tier: "pull",
   config: ["method", "url", "reqHeaders", "body", "connection"],
@@ -4116,7 +4190,7 @@ LEGACY_TESTS.single = {
     const raws           = [], terms           = [];
     for (const r of ress) {
       if (!r) continue;
-      raws.push(r.transcript?.[r.transcript.length - 1]?.got ?? r.raw ?? "");
+      raws.push(gradedText(r));
       terms.push(...(r.terms || []));
     }
     const s = scoreSingle(t, terms, raws, !(kind != null && OUTPUT_KINDS[kind]?.terms));
@@ -4181,11 +4255,32 @@ function metricsProblems(list         , at        , bad          )       {
 /** Whether any of [list] asks a grader. */
 const gradedIn = (list         )          => Array.isArray(list) && list.some((m           ) => !!METRICS[m?.type]?.graded);
 
+/** Whether [error] is a reply the job's rules threw away, as the Discarded
+    metric reads it: a result that came back, where any other error is a
+    request that never did. */
+const thrownAway = (error                           )          => !!error && error.startsWith("discarded: ");
+
+/** [s] as an errored result's score: whatever its metrics read, a result
+    with no reply fails, and only a discarded reply is left to them (#544) --
+    or a refused call, to a case that scores whether it was (#559). */
+function erroredScore                              (s   , error                           )    {
+  const asksRefusal = !!s?.metrics?.some(r => r.weight !== 0 && METRICS[r.type]?.readsRefusal);
+  return s && error != null && !thrownAway(error) && !asksRefusal ? { ...s, pass: false } : s;
+}
+
+/** The reply a metric grades and Results shows (#546): the last stage's
+    reply after its Pre-format filters, or the raw reply. */
+function gradedText(res                                                      )         {
+  const last = res?.transcript?.at(-1);
+  return last?.filtered?.got ?? last?.got ?? res?.raw ?? "";
+}
+
 /** What a metric reads of one result. */
 function metricInput(res                    , kase             , more          )              {
   const last = res.transcript?.at(-1);
-  const text = last?.got ?? res.raw ?? "";
-  return { text, said: last?.said ?? null, terms: res.terms ?? [], replies: [text], plain: !!more.plain,
+  const text = gradedText(res);
+  // `of: "said"` reads the reply as it came back, before any filter.
+  return { text, said: last?.said ?? (last?.filtered ? last.got ?? null : null), terms: res.terms ?? [], replies: [text], plain: !!more.plain,
            asked: last?.sent ?? null,
            // A case that has been given its own expected reply ("Take B as
            // expected") is compared with that; otherwise the Source's recorded
@@ -4201,7 +4296,7 @@ function runInput(ress                                       , plain         )
   let ms = 0;
   for (const r of ress) {
     if (!r) continue;
-    replies.push(r.transcript?.[r.transcript.length - 1]?.got ?? r.raw ?? "");
+    replies.push(gradedText(r));
     terms.push(...(r.terms || []));
     ms += r.ms ?? 0;
   }
@@ -4217,6 +4312,19 @@ function readMetric(m        , input             , ctx           )
   const label = (isStr(m.metric) && m.metric.trim()) || entry?.label || m.type;
   if (!entry) return { type: m.type, label, weight, pass: false, score: 0, reason: `"${m.type}" is not a metric this lab has` };
   const read = m.of === "said" ? { ...input, text: input.said ?? input.text } : input;
+  // An errored result has no reply to read (#544): a metric that reads one
+  // fails on the error, and a grader is never asked about the empty reply.
+  if (input.error != null && !(entry.readsResult && (thrownAway(input.error) || entry.readsRefusal))) {
+    const fail                = { type: m.type, label, weight, pass: false, score: 0, reason: input.error };
+    if (entry.graded) return fail;
+    // What it would have missed still counts toward a run's totals.
+    let o                                   = null;
+    try { o = entry.score(read, m, {}); } catch { o = null; }
+    if (!o) return null;
+    if (typeof (o                    ).then === "function") return fail;
+    const { found, missed, invented } = o                 ;
+    return { ...fail, ...(found ? { found } : {}), ...(missed ? { missed } : {}), ...(invented ? { invented } : {}) };
+  }
   const done = (o                      )                       => {
     if (!o) return null;
     const turned = m.not ? { ...o, pass: !o.pass, score: entry.counts ? -o.score : 1 - o.score, reason: `not: ${o.reason}` } : o;
@@ -4249,6 +4357,18 @@ function readMetric(m        , input             , ctx           )
   } catch (e) { return timed(failed(e)); }
 }
 
+/** What stops [grader] being asked, where the lookups can say: a grader is
+    asked words, and needs a model to ask. */
+function graderProblems(grader            , ctx               , bad          )       {
+  const p = grader && ctx.profiles ? ctx.profiles(grader.id) : null;
+  if (grader && ctx.profiles && !p) bad.push(`Grader ${grader.name || grader.id} not found`);
+  else if (p) {
+    const type = CONNECTION_TYPES[typeOf(p            )];
+    if (type && !(type.answers ?? ["prompt"]).includes("prompt")) bad.push(`the grader has to be asked words, and ${type.label} is not`);
+    else if (!type?.local && !String((p            ).model || "").trim()) bad.push("no model on the grader's connection");
+  }
+}
+
 /** An eval's score from its metrics' readings, the way [mode] says: all must
     pass, or weighted points against [threshold]. What a case metric found,
     missed and invented is carried up, for a run's totals. Null for none. */
@@ -4277,7 +4397,7 @@ async function readMetrics(list                   , input             , ctx
     const r = await readMetric(m, input, ctx);
     if (r) metrics.push(r);
   }
-  return scoreOf(metrics, mode, threshold);
+  return erroredScore(scoreOf(metrics, mode, threshold), input.error);
 }
 
 /** [list] read over a whole run, synchronously: each metric over the run's
@@ -4331,14 +4451,7 @@ LEGACY_TESTS.metrics = {
     // The eval's own grader, or the lab's.
     const grader = isRef(t.grader) ? t.grader : isRef(ctx.grader) ? ctx.grader : null;
     if (gradedIn(t.metrics) && !grader) bad.push("a model-graded metric needs a grader: pick one on the eval, or make a connection the lab's grader");
-    // A grader is asked words, and needs a model to ask.
-    const p = grader && ctx.profiles ? ctx.profiles(grader.id) : null;
-    if (grader && ctx.profiles && !p) bad.push(`Connection ${grader.name || grader.id} not found`);
-    else if (p) {
-      const type = CONNECTION_TYPES[typeOf(p            )];
-      if (type && !(type.answers ?? ["prompt"]).includes("prompt")) bad.push(`the grader has to be asked words, and ${type.label} is not`);
-      else if (!type?.local && !String((p            ).model || "").trim()) bad.push("no model on the grader's connection");
-    }
+    graderProblems(grader, ctx, bad);
   },
   rules: t => (Array.isArray(t.metrics) ? t.metrics : []).map((m        , x        ) => ({
     key: `m${x}`, label: (isStr(m.metric) && m.metric) || METRICS[m.type]?.label || m.type,
@@ -4450,6 +4563,11 @@ EVAL_TYPES.group = {
       if (held && t.pin != null && typeof held.versions === "number" && t.pin > held.versions) {
         bad.push(`${g.name || g.id} has no version ${t.pin}`);
       }
+      // The grader the group asks: the run's copy of the link carries it, a
+      // pipeline's reads it from the group as the lab lists it. One that is
+      // gone would fail every item on a metric that cannot run (#545).
+      const grader = isRef(t.grader) ? t.grader : isRef(held?.grader) ? held .grader        : null;
+      graderProblems(grader, ctx, bad);
       return;
     }
     if (!isObj(own)) return void bad.push("an eval links a Library group, or holds its own");
@@ -4466,14 +4584,7 @@ EVAL_TYPES.group = {
     // The group's own grader, or the lab's.
     const grader = isRef(own.grader) ? own.grader : isRef(ctx.grader) ? ctx.grader : null;
     if (gradedIn(own.every) && !grader) bad.push("a model-graded metric needs a grader: pick one on the eval, or make a connection the lab's grader");
-    // A grader is asked words, and needs a model to ask.
-    const p = grader && ctx.profiles ? ctx.profiles(grader.id) : null;
-    if (grader && ctx.profiles && !p) bad.push(`Connection ${grader.name || grader.id} not found`);
-    else if (p) {
-      const type = CONNECTION_TYPES[typeOf(p            )];
-      if (type && !(type.answers ?? ["prompt"]).includes("prompt")) bad.push(`the grader has to be asked words, and ${type.label} is not`);
-      else if (!type?.local && !String((p            ).model || "").trim()) bad.push("no model on the grader's connection");
-    }
+    graderProblems(grader, ctx, bad);
   },
   rules: t => ownList(t).map((m        , x        ) => ({
     key: `m${x}`, label: (isStr(m.metric) && m.metric) || METRICS[m.type]?.label || m.type,
@@ -4864,8 +4975,12 @@ STEP_TYPES.httpRequest = {
       }
     }
     if (isStr(step.prompt)) {
-      try { api.place(step.body, step        ); }
-      catch (e) { bad.push(`${at}: ${(e         ).message}`); }
+      // Built here as each call builds it, so an expression none of them
+      // could read is refused before the run rather than failing every call.
+      try {
+        const why = templateProblem([step.path, step.query, api.place(step.body, step        )]);
+        if (why) bad.push(`${at}: the request could not be built -- ${why}`);
+      } catch (e) { bad.push(`${at}: ${(e         ).message}`); }
     }
   },
 };
@@ -4890,7 +5005,6 @@ STEP_TYPES.echo = {
 // the flow reads one. No profile, no prompt -- the record is the reply.
 STEP_TYPES.recorded = {
   label: "Recorded", slot: "target", asks: "nothing", local: true, replaces: "recorded",
-  description: "Pre-recorded from previous workflow runs, replays only.",
   firstJobOnly: "a call's recorded reply is job 1's",
   in: "item", out: "text",
   apply: "runPipeline",
@@ -6082,6 +6196,10 @@ function validatePipeline(input         , ctx          = {})           {
     ? (c.sources?.(content.ref?.id)?.files ?? content.files ?? null) : null;
   const nonText = (contentFiles || []).map(f => String((f                             )?.name ?? f))
     .find(n => !/\.(?:txt|md|csv)$/i.test(n));
+  // A document goes whole to the API, so only a type that reads one is
+  // handed it -- never sent as an image or as text it is not.
+  const docType = (contentFiles || []).map(f => documentTypeOf(String((f                             )?.name ?? f)))
+    .find(Boolean) ?? null;
   targets.forEach((_, i) => {
     const conns = doc.jobs.map((_     , k        ) => (targetEntryOf(doc, i, k)?.local ? localConnOf(targetEntryOf(doc, i, k)) : lookup(targetAnswerOf(doc, i, k) )));
     conns.forEach((p         , k        ) => {
@@ -6111,6 +6229,10 @@ function validatePipeline(input         , ctx          = {})           {
         const name = targetAnswerOf(doc, i, k)?.connection.name || type .label;
         bad.push(`${n > 1 ? `${jobLabel(doc, k)}, ` : ""}${targetLabel(doc, i)}: ${name} can't send an ${HTTP_APIS[api]?.label ?? api} request`);
       }
+      if (docType && type && !type.documents && sendsImage(doc.jobs[k])) {
+        const name = targetAnswerOf(doc, i, k)?.connection.name || type.label;
+        bad.push(`${n > 1 ? `${jobLabel(doc, k)}, ` : ""}${targetLabel(doc, i)}: ${name} can't read ${docType.label}s`);
+      }
     });
     // The model is the profile's unless the target's step carries it.
     const bare = conns.findIndex((p                           , k        ) => p && c.profiles && !local(p)
@@ -6913,7 +7035,9 @@ function scenarioEvals(run             , i        , items
   return evals.map((t, j) => {
     const type = EVAL_TYPES[t.type];
     const whole = isWholeRun(type, t);
-    const base = { id: t.id, label: evalLabel({ evals }, j), whole, skipped: skipRest,
+    // An unnamed eval that links a group goes by the group's name, as the
+    // Evals Summary names it, not "Eval 1" (#519).
+    const base = { id: t.id, label: (t.name || "").trim() || t.group?.name || evalLabel({ evals }, j), whole, skipped: skipRest,
                    verdict: null                          , rules: type?.rules?.(t) ?? [], items: cells.map(() => null)                        ,
                    ran: 0, passed: 0, skippedItems: 0 };
     if (skipRest) {
@@ -7053,6 +7177,11 @@ function validateEvals(ev           , files
       if (scored.length > 1) say("expects its answer discarded, and states something the answer should hold too");
       continue;
     }
+    if (scored.some(m => m.type === "refused" && !m.not)) {
+      // Nor does a call that was refused: there is no answer.
+      if (scored.length > 1) say("expects its call refused, and states something the answer should hold too");
+      continue;
+    }
     const wants = scored.filter(m => !m.not && (m.type === "contains-all" || m.type === "contains-any"));
     const forbids = scored.filter(m => m.not && m.type === "contains");
     for (const m of wants) for (const t of values(m)) if (!matchable(t)) say(`expects ${t}, which its own scorer cannot match`);
@@ -7166,7 +7295,7 @@ export {
     loopReplyError, preparedSize,
     termIn, forbiddenIn, readCase, gradedSetFrom, caseItem, caseMetrics, metricLines, upgradeDatasetBody, takeAsExpected, readGroup, readGroupRun, datasetRules, upgradeResults, emptyTally, addToTally, isHostedUrl,
     SEED, REPLY_TOKENS_BEFORE, ANTHROPIC_MAX_TOKENS, asNumber, pinReplyTokens, mappingsFor,
-    CONNECTION_TYPES, HTTP_APIS, httpApiFor, httpRequestOf, httpReplyOf, readFlowReply, WdlError, localAnswer, typeOf, profileType, convertProfile, splitOllama, DECODING_KEYS, apiBase,
+    CONNECTION_TYPES, DOCUMENT_TYPES, documentTypeOf, documentUrl, attachmentOf, HTTP_APIS, httpApiFor, httpRequestOf, httpReplyOf, readFlowReply, WdlError, localAnswer, typeOf, listedConnection, profileType, convertProfile, splitOllama, DECODING_KEYS, apiBase,
     EDGE_448, budgetLabel, IMAGE_FORMATS, encoderQuality,
     tallyPercent, runPipeline, validateEvals, evalsWarnings, evalsJson, readList,
     scoreSingle, singleRules, parseCount, lengthHolds, countHolds, COUNT_OPS, LENGTH_OPS,
@@ -7180,7 +7309,7 @@ CONNECTION_FIELDS, SETTING_KEYS, OPTION_FIELDS, jobLabel, scenarioLabel, jobWord
     contentOf, withContent, replyOf,
     targetsOf, targetLetter, targetLabel, targetStepOf, targetEntryOf, jobTargetType, targetAnswerOf, targetAnswerKey, recordedTargetOf, withTarget,
     tokensOf, withTokens, sendsImage, withImage, flowStepOf, requestStepOf, readAsOf, outOf, withOut, slotEntry, SLOTS,
-    filterSetsOf, withFilterSets, stageModifiers, beforeFilters,
+    filterSetsOf, withFilterSets, stageModifiers, beforeFilters, gradedText,
     blankPipeline, upgradePipeline, fatProfileRef, newId, newEval, newLink, validatePipeline, resolvePipeline, pipelineOfRun, stagesFor, profileIds, answerKeys,
     scenarioProfile, lastKind, lastModifier, modifierSummary, itemScores, scenarioEvals, scenarioPasses, groupPass, overallVerdict, passVerdict, isSkipped, failedScore,
     evalLabel, evalsOf, evalsDataset, casesRef, groupOf, groupOfMetrics, pipelineWarnings, itemsFound, previewOf, isWholeRun, EVAL_FIELDS,

package/lab/flows/wdl.mjs

@@ -363,6 +363,25 @@ function endOf(s        , from        )         {
   throw new WdlError("an @{ is never closed");
 }
 
+/** Why [template] cannot be evaluated whatever it is evaluated against -- an
+    expression in it that does not parse -- or null. Read before a run, so a
+    template every call would fail on is refused rather than sent. */
+export function templateProblem(template         )                {
+  try { walk(template); return null; }
+  catch (e) { if (e instanceof WdlError) return e.message; throw e; }
+  function walk(v         )       {
+    if (Array.isArray(v)) return v.forEach(walk);
+    if (isObj(v)) return Object.values(v).forEach(walk);
+    if (typeof v !== "string" || v.startsWith("@@")) return;
+    if (v.startsWith("@") && !v.startsWith("@{")) return void parse(v.slice(1));
+    for (let open = v.indexOf("@{"); open >= 0; open = v.indexOf("@{", open)) {
+      const close = endOf(v, open + 2);
+      parse(v.slice(open + 2, close));
+      open = close + 1;
+    }
+  }
+}
+
 /** [template] with every expression in it evaluated: each string that is
     one, or holds one, wherever it sits in objects and arrays. */
 export function evaluate(template         , ctx            )          {

package/lab/kinds/list.mjs

@@ -293,6 +293,8 @@ export const FILTER_SEPARATORS = ["comma", "lines", "semicolon", "none"]
                                                                             
                                         
                    
+                                                                         
+                     
  
 
 /** A base filter type (#334's decision: a registry, not an enum). A reader
@@ -471,19 +473,20 @@ export function applyFilter(filter        , reply             )                {
 export function applyFilterSet(filters                   , reply             , stage                   )                {
   const instruction = reply.instruction ?? "";
   let body = reply.body, items = reply.items;
-  const dropped            = [];
+  const dropped            = [], rewrote           = [];
   let emptied                    ;
   for (const filter of filters) {
     const scope = FILTER_TYPES[filter.type]?.scope;
     if (stage && scope && (scope === "text") !== (stage === "text")) continue;
     const out = applyFilter(filter, { body, items, instruction });
+    if (out.body !== body) rewrote.push(filter.id);
     body = out.body;
     if (items.length && !out.items.length) emptied = filter.id;
     items = out.items;
     dropped.push(...out.dropped);
     if (out.reject != null) return { body, items, dropped, reject: out.reject };
   }
-  return { body, items, dropped, reject: null, ...(emptied ? { emptied } : {}) };
+  return { body, items, dropped, reject: null, ...(emptied ? { emptied } : {}), ...(rewrote.length ? { rewrote } : {}) };
 }
 
 // A newline always ends an item, as it does in the real csv/lines parse