evals-lab 0.11.0 → 0.13.0

package/lab/evals-core.mjs

@@ -34,7 +34,7 @@
 
 import yaml from "./js-yaml.mjs";
                                                                                                                                                  
-import { evaluate, asText, WdlError,                 } from "./flows/wdl.mjs";
+import { evaluate, asText, templateProblem, WdlError,                 } from "./flows/wdl.mjs";
 import { stepsOf as flowStepsOf, actionsByName, readerFor, readsOf,                          } from "./flows/record.mjs";
 import { flowApi,              } from "./flows/flowApi.mjs";
 import registerMetrics from "./metrics/builtin.mjs";
@@ -166,13 +166,11 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
 /** A target step that sends a Power Automate step's request
     (docs/power-automate.md § The HTTP Request step): the flow's own template
     -- method, path, query and body, Workflow Definition Language expressions
-    and all -- with the target's words and fields placed in the body by its
-    HTTP API, evaluated against each item's record. The profile gives the
-    address and the key. */
+    and all -- with the target's words and fields placed in the body by the
+    job's HTTP API (its flow step's), evaluated against each item's record.
+    The profile gives the address and the key. */
                                                
                       
-                                                    
-              
                  
                
                                 
@@ -240,7 +238,7 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
  
 
 /** What a target sends in one job: one of the target step types. */
-                                                                                                                                
+                                                                                                                        
 
 /** A target (pipeline-model §16): one column through every job, compared
     side by side in Results -- what version 9 called a scenario. Its step in
@@ -609,6 +607,9 @@ export const FILTER_STAGE_LABEL                              = { text: "Pre-form
                                                                             
                                               
                 
+                                                                            
+                                                                         
+                                           
  
 
 /** How much of one text a run keeps -- a reply's body, a grader's prompt or
@@ -899,6 +900,14 @@ export function kept(text        , max = KEPT_CHARS)         {
                     
                                                                                     
                        
+                                                                              
+                                                                               
+                                                                          
+                        
+                                                                          
+                                                                        
+                       
+                         
                                                                              
                                                                            
                  
@@ -1692,9 +1701,17 @@ function datasetRules(body         )                  {
                                                                                                   
                                     
                           
+                                                                             
+                                                                             
+                                                            
+                      
                                                                            
                                  
                     
+                                                                             
+                                                                             
+                                                                               
+                                        
                                                                           
                                                                                
                                                                          
@@ -1719,6 +1736,10 @@ function datasetRules(body         )                  {
                                                     
                                                                                                             
                                                         
+                                                                             
+                                                                          
+                                                                   
+                                         
                                                                                    
                                 
                                                                             
@@ -1991,6 +2012,35 @@ function asNumber(v         )                {
 // a tuned local engine is a llama.cpp profile with its settings.
 const CONNECTION_TYPES                                 = Object.create(null);
 
+// ---- documents (#558) ------------------------------------------------------
+// A file a model's API reads whole, sent unprocessed: the lab does not parse,
+// render or extract it. It travels where an image does -- the item's
+// attachment, a data URL -- carrying its name, which some APIs want beside
+// the bytes. Only a connection type marked `documents` is handed one.
+const DOCUMENT_TYPES                                                   = {
+  ".pdf": { media: "application/pdf", label: "PDF" },
+};
+
+/** The DOCUMENT_TYPES entry a file's name is, or null. */
+function documentTypeOf(name        )                                          {
+  const dot = String(name).lastIndexOf(".");
+  return dot < 0 ? null : DOCUMENT_TYPES[String(name).slice(dot).toLowerCase()] ?? null;
+}
+
+/** A document's bytes, base64, as the attachment a stage sends. */
+function documentUrl(name        , media        , base64        )         {
+  return `data:${media};name=${encodeURIComponent(name)};base64,${base64}`;
+}
+
+/** An attachment's media type, bytes and name (null for an image, which
+    carries none), or null for anything that is not a base64 data URL. */
+function attachmentOf(dataUrl        )                                                              {
+  const m = /^data:([^;,]+)((?:;[^;,=]+=[^;,]*)*);base64,(.*)$/s.exec(dataUrl);
+  if (!m) return null;
+  const name = /;name=([^;,]*)/.exec(m[2] )?.[1];
+  return { media: m[1] , data: m[3] , name: name == null ? null : decodeURIComponent(name) };
+}
+
 /** The key [key] as the header most endpoints expect, when there is one. */
 const bearer = (key                           )                         => (key ? { Authorization: "Bearer " + key } : {});
 
@@ -2103,6 +2153,13 @@ CONNECTION_TYPES.recorded = {
   request: sendsNothing, parseReply: sendsNothing, listModels: sendsNothing, parseModels: () => [],
 };
 
+/** Whether Setup and the connection pickers list a stored connection. A local
+    type's (Echo, Recorded) is a target step's stand-in now (`localSteps`), so
+    a row of one is kept only for the pipelines and runs that name it. */
+function listedConnection(conn                           )          {
+  return !CONNECTION_TYPES[typeOf(conn                               )]?.local;
+}
+
 /** A local type's answer to one stage, as a model's would come back. An item
     with no text of its own (Prompt only) is answered from the prompt, and
     that reply is read against nothing -- the prompt is the reply, not an
@@ -2118,9 +2175,19 @@ function localAnswer(conn          , text               , sent        , record
 const chatSchema = (body                         , schema                         )                          =>
   ({ ...body, response_format: { type: "json_schema", json_schema: { name: "reply", strict: true, schema } } });
 
+/** An item's attachment as a chat completions content part: an image as
+    image_url, a document as a file part, named, its bytes a data URL. */
+function chatAttachment(dataUrl        )         {
+  const a = attachmentOf(dataUrl);
+  if (!a || a.media.startsWith("image/")) return { type: "image_url", image_url: { url: dataUrl } };
+  return { type: "file", file: { filename: a.name ?? "document", file_data: `data:${a.media};base64,${a.data}` } };
+}
+
 CONNECTION_TYPES["openai-compatible"] = {
   id: "openai-compatible", label: "OpenAI-compatible",
   description: "OpenAI, OpenRouter, vLLM, LM Studio: any chat completions endpoint.",
+  // A hosted provider wants a key; a server of your own may not.
+  keyOptional: (conn) => !isHostedUrl(String(conn.url ?? "")),
   settings: [
     // A reasoning model refuses a temperature.
     { key: "temperature", label: "Temperature", control: "text", default: "", hint: "",
@@ -2143,11 +2210,14 @@ CONNECTION_TYPES["openai-compatible"] = {
         ...(predict == null ? {} : hosted ? { max_completion_tokens: predict } : { max_tokens: predict }),
         messages: [{ role: "user", content: [
           { type: "text", text: prompt },
-          ...(dataUrl ? [{ type: "image_url", image_url: { url: dataUrl } }] : []),
+          ...(dataUrl ? [chatAttachment(dataUrl)] : []),
         ] }],
       },
     };
   },
+  // OpenAI's file part; a compatible server that has none refuses the call,
+  // and the refusal is that item's error.
+  documents: true,
   structured: chatSchema,
   parseReply: chatReply,
   listModels, parseModels: idsFrom,
@@ -2170,16 +2240,17 @@ CONNECTION_TYPES.anthropic = {
   // Asked words (a Prompt step, a grader) or sent a flow's whole request (an
   // HTTP Request step): one profile and one key do both.
   answers: ["prompt", "request"],
+  speaks: () => "anthropic",
   // The Messages API: the x-api-key and anthropic-version headers, base64
-  // image content blocks, and /v1/models for the list.
+  // image and document content blocks, and /v1/models for the list.
+  documents: true,
   request(conn, prompt, dataUrl, base, key) {
     const temp = asNumber(conn.temperature);
     const predict = asNumber(conn.maxTokens) ?? ANTHROPIC_MAX_TOKENS;
     const content           = [];
-    if (dataUrl) {
-      const m = /^data:(image\/[^;]+);base64,(.*)$/s.exec(dataUrl);
-      if (m) content.push({ type: "image", source: { type: "base64", media_type: m[1], data: m[2] } });
-    }
+    const a = dataUrl ? attachmentOf(dataUrl) : null;
+    if (a?.media.startsWith("image/")) content.push({ type: "image", source: { type: "base64", media_type: a.media, data: a.data } });
+    else if (a) content.push({ type: "document", source: { type: "base64", media_type: a.media, data: a.data } });
     content.push({ type: "text", text: prompt });
     return {
       url: base + "/messages",
@@ -2224,6 +2295,7 @@ const SPEED_IMAGE = "before", SPEED_CACHE = "off", SPEED_THINKING = "off";
 CONNECTION_TYPES["llama.cpp"] = {
   id: "llama.cpp", label: "llama.cpp",
   description: "A llama-server you run, with its sampler settings.",
+  keyOptional: () => true,
   settings: [
     { key: "temperature", label: "Temperature", control: "text", default: "", hint: "" },
     { key: "seed", label: "Seed", control: "text", default: "", hint: "" },
@@ -2501,8 +2573,18 @@ CONNECTION_TYPES.http = {
     { key: "headers", label: "Headers", control: "text", default: "", hint: "", scope: "connection" },
     // What Test connection asks of it: a GET, keyed as a run is.
     { key: "testPath", label: "Test path", control: "text", default: "", hint: "", scope: "connection" },
+    // Read off the address (httpApiFor); set by hand for one the lab cannot
+    // tell, such as a gateway in front of a provider.
+    { key: "requestFormat", label: "Request format", control: "select", default: "", hint: "", scope: "connection",
+      choices: [{ value: "", label: "From address" },
+                ...Object.entries(HTTP_APIS).map(([value, a]) => ({ value, label: a.label }))] },
   ],
   answers: ["request"],
+  keyOptional: (conn) => conn.auth === "none",
+  speaks(conn) {
+    const api = String(conn.requestFormat || "") || httpApiFor(originBase(conn.url));
+    return HTTP_APIS[api] && api !== "raw" ? api : null;
+  },
   lists: false,
   baseOf: originBase,
   send(conn, req, base, key) {
@@ -2785,7 +2867,7 @@ function readCase(kase      , res                                   , plain = fa
     }
     return r ? [r                 ] : [];
   });
-  const s = scoreOf(metrics) ?? { pass: true, score: null, metrics };
+  const s = erroredScore(scoreOf(metrics), res.error) ?? { pass: !res.error, score: null, metrics };
   const counted = metrics.filter(r => r.weight !== 0);
   return { ...s, metrics, found: s.found ?? [], missed: s.missed ?? [], invented: s.invented ?? [],
            discarded: !!res.error, count: res.error ? 0 : (res.terms || []).length,
@@ -3183,6 +3265,7 @@ async function runPipeline(stages                  , dataUrl               , cal
     const pre = applyFilterSet(st.before || [], { body: res.raw ?? "", items: [], instruction: against }, "text");
     const by = (filter        , stage             , rejected         )             => ({ filter, stage, job: i + 1, rejected });
     if (pre.reject != null) return result({ error: named(`discarded: ${pre.reject}`), filteredBy: by(pre.reject, "text", true) });
+    if (pre.rewrote) transcript[transcript.length - 1] .filtered = { got: pre.body, by: pre.rewrote };
     const read = reader.read(pre.body, { echo: against, parsing: opts.parsing, settings: st.settings });
     if (read.error) return result({ error: named(read.error) });
     const mod = applyModifiers(mods[i], kind, read.value, { instruction: against, body: read.body ?? res.raw ?? "" });
@@ -3297,7 +3380,7 @@ function applyModifiers   (list                                 , kind        ,
 // 16: a target names an AI Connection and, optionally, a Connection Profile,
 // where it named one Target profile; a run's table is keyed by the pair
 // (answerKey) -- the Target profile split in the store as it opens (#445).
-const PIPELINE_VERSION = 16         ;
+const PIPELINE_VERSION = 17         ;
 
 // Plain objects, so an entry is added by assignment and a reader never needs
 // to know which registered it.
@@ -3719,16 +3802,17 @@ function onlyFields(obj        , at        , allowed                   , bad
 
 // A Source is referenced, with the two numbers that decide how many items a
 // run over it is: the first N files, and how many times over.
-// A folder of files: images sent as images, text files as {text}. The
-// extensions are server.py's FILE_TYPES, which the server holds uploads to.
+// A folder of files: images sent as images, documents whole (DOCUMENT_TYPES),
+// text files as {text}. The extensions are server.py's FILE_TYPES, which the
+// server holds uploads to.
 const FILE_LIBRARY_TEXT = [".txt", ".md", ".csv"]         ;
 SOURCE_TYPES.files = {
   label: "File Library",
   icon: "folder",
-  description: "A folder of images and text files you upload.",
+  description: "A folder of images, PDFs and text files you upload.",
   group: "Files",
   noun: "file",
-  uploads: [".jpg", ".jpeg", ".png", ".heic", ".heif", ".dng", ".tif", ".tiff", ...FILE_LIBRARY_TEXT],
+  uploads: [".jpg", ".jpeg", ".png", ".heic", ".heif", ".dng", ".tif", ".tiff", ...Object.keys(DOCUMENT_TYPES), ...FILE_LIBRARY_TEXT],
   sendsImage: true,
   hasActions: false,
   itemText: name => FILE_LIBRARY_TEXT.some(ext => name.toLowerCase().endsWith(ext)),
@@ -3745,7 +3829,9 @@ SOURCE_TYPES.workflow = {
   icon: "workflow",
   description: "A cloud flow, with records of its steps' calls from its run history.",
   group: "Workflows",
-  noun: "record",
+  // Each item reads as a call, as the Source's Calls group names them, though
+  // the file holding one is a record.
+  noun: "call",
   uploads: [".json"],
   sendsImage: false,
   hasActions: true,
@@ -3834,7 +3920,7 @@ const API_METHODS = ["GET", "POST", "PUT", "PATCH", "DELETE"];
 DYNAMIC_PROVIDERS["api"] = {      // provider: the API call adapter's own id
   id: "api",                      // provider: its id, the row's config.provider
   label: "API call",
-  icon: "square-code",
+  icon: "arrow-left-right",
   noun: "call",
   tier: "pull",
   config: ["method", "url", "reqHeaders", "body", "connection"],
@@ -4104,7 +4190,7 @@ LEGACY_TESTS.single = {
     const raws           = [], terms           = [];
     for (const r of ress) {
       if (!r) continue;
-      raws.push(r.transcript?.[r.transcript.length - 1]?.got ?? r.raw ?? "");
+      raws.push(gradedText(r));
       terms.push(...(r.terms || []));
     }
     const s = scoreSingle(t, terms, raws, !(kind != null && OUTPUT_KINDS[kind]?.terms));
@@ -4169,11 +4255,32 @@ function metricsProblems(list         , at        , bad          )       {
 /** Whether any of [list] asks a grader. */
 const gradedIn = (list         )          => Array.isArray(list) && list.some((m           ) => !!METRICS[m?.type]?.graded);
 
+/** Whether [error] is a reply the job's rules threw away, as the Discarded
+    metric reads it: a result that came back, where any other error is a
+    request that never did. */
+const thrownAway = (error                           )          => !!error && error.startsWith("discarded: ");
+
+/** [s] as an errored result's score: whatever its metrics read, a result
+    with no reply fails, and only a discarded reply is left to them (#544) --
+    or a refused call, to a case that scores whether it was (#559). */
+function erroredScore                              (s   , error                           )    {
+  const asksRefusal = !!s?.metrics?.some(r => r.weight !== 0 && METRICS[r.type]?.readsRefusal);
+  return s && error != null && !thrownAway(error) && !asksRefusal ? { ...s, pass: false } : s;
+}
+
+/** The reply a metric grades and Results shows (#546): the last stage's
+    reply after its Pre-format filters, or the raw reply. */
+function gradedText(res                                                      )         {
+  const last = res?.transcript?.at(-1);
+  return last?.filtered?.got ?? last?.got ?? res?.raw ?? "";
+}
+
 /** What a metric reads of one result. */
 function metricInput(res                    , kase             , more          )              {
   const last = res.transcript?.at(-1);
-  const text = last?.got ?? res.raw ?? "";
-  return { text, said: last?.said ?? null, terms: res.terms ?? [], replies: [text], plain: !!more.plain,
+  const text = gradedText(res);
+  // `of: "said"` reads the reply as it came back, before any filter.
+  return { text, said: last?.said ?? (last?.filtered ? last.got ?? null : null), terms: res.terms ?? [], replies: [text], plain: !!more.plain,
            asked: last?.sent ?? null,
            // A case that has been given its own expected reply ("Take B as
            // expected") is compared with that; otherwise the Source's recorded
@@ -4189,7 +4296,7 @@ function runInput(ress                                       , plain         )
   let ms = 0;
   for (const r of ress) {
     if (!r) continue;
-    replies.push(r.transcript?.[r.transcript.length - 1]?.got ?? r.raw ?? "");
+    replies.push(gradedText(r));
     terms.push(...(r.terms || []));
     ms += r.ms ?? 0;
   }
@@ -4205,6 +4312,19 @@ function readMetric(m        , input             , ctx           )
   const label = (isStr(m.metric) && m.metric.trim()) || entry?.label || m.type;
   if (!entry) return { type: m.type, label, weight, pass: false, score: 0, reason: `"${m.type}" is not a metric this lab has` };
   const read = m.of === "said" ? { ...input, text: input.said ?? input.text } : input;
+  // An errored result has no reply to read (#544): a metric that reads one
+  // fails on the error, and a grader is never asked about the empty reply.
+  if (input.error != null && !(entry.readsResult && (thrownAway(input.error) || entry.readsRefusal))) {
+    const fail                = { type: m.type, label, weight, pass: false, score: 0, reason: input.error };
+    if (entry.graded) return fail;
+    // What it would have missed still counts toward a run's totals.
+    let o                                   = null;
+    try { o = entry.score(read, m, {}); } catch { o = null; }
+    if (!o) return null;
+    if (typeof (o                    ).then === "function") return fail;
+    const { found, missed, invented } = o                 ;
+    return { ...fail, ...(found ? { found } : {}), ...(missed ? { missed } : {}), ...(invented ? { invented } : {}) };
+  }
   const done = (o                      )                       => {
     if (!o) return null;
     const turned = m.not ? { ...o, pass: !o.pass, score: entry.counts ? -o.score : 1 - o.score, reason: `not: ${o.reason}` } : o;
@@ -4237,6 +4357,18 @@ function readMetric(m        , input             , ctx           )
   } catch (e) { return timed(failed(e)); }
 }
 
+/** What stops [grader] being asked, where the lookups can say: a grader is
+    asked words, and needs a model to ask. */
+function graderProblems(grader            , ctx               , bad          )       {
+  const p = grader && ctx.profiles ? ctx.profiles(grader.id) : null;
+  if (grader && ctx.profiles && !p) bad.push(`Grader ${grader.name || grader.id} not found`);
+  else if (p) {
+    const type = CONNECTION_TYPES[typeOf(p            )];
+    if (type && !(type.answers ?? ["prompt"]).includes("prompt")) bad.push(`the grader has to be asked words, and ${type.label} is not`);
+    else if (!type?.local && !String((p            ).model || "").trim()) bad.push("no model on the grader's connection");
+  }
+}
+
 /** An eval's score from its metrics' readings, the way [mode] says: all must
     pass, or weighted points against [threshold]. What a case metric found,
     missed and invented is carried up, for a run's totals. Null for none. */
@@ -4265,7 +4397,7 @@ async function readMetrics(list                   , input             , ctx
     const r = await readMetric(m, input, ctx);
     if (r) metrics.push(r);
   }
-  return scoreOf(metrics, mode, threshold);
+  return erroredScore(scoreOf(metrics, mode, threshold), input.error);
 }
 
 /** [list] read over a whole run, synchronously: each metric over the run's
@@ -4319,14 +4451,7 @@ LEGACY_TESTS.metrics = {
     // The eval's own grader, or the lab's.
     const grader = isRef(t.grader) ? t.grader : isRef(ctx.grader) ? ctx.grader : null;
     if (gradedIn(t.metrics) && !grader) bad.push("a model-graded metric needs a grader: pick one on the eval, or make a connection the lab's grader");
-    // A grader is asked words, and needs a model to ask.
-    const p = grader && ctx.profiles ? ctx.profiles(grader.id) : null;
-    if (grader && ctx.profiles && !p) bad.push(`Connection ${grader.name || grader.id} not found`);
-    else if (p) {
-      const type = CONNECTION_TYPES[typeOf(p            )];
-      if (type && !(type.answers ?? ["prompt"]).includes("prompt")) bad.push(`the grader has to be asked words, and ${type.label} is not`);
-      else if (!type?.local && !String((p            ).model || "").trim()) bad.push("no model on the grader's connection");
-    }
+    graderProblems(grader, ctx, bad);
   },
   rules: t => (Array.isArray(t.metrics) ? t.metrics : []).map((m        , x        ) => ({
     key: `m${x}`, label: (isStr(m.metric) && m.metric) || METRICS[m.type]?.label || m.type,
@@ -4438,6 +4563,11 @@ EVAL_TYPES.group = {
       if (held && t.pin != null && typeof held.versions === "number" && t.pin > held.versions) {
         bad.push(`${g.name || g.id} has no version ${t.pin}`);
       }
+      // The grader the group asks: the run's copy of the link carries it, a
+      // pipeline's reads it from the group as the lab lists it. One that is
+      // gone would fail every item on a metric that cannot run (#545).
+      const grader = isRef(t.grader) ? t.grader : isRef(held?.grader) ? held .grader        : null;
+      graderProblems(grader, ctx, bad);
       return;
     }
     if (!isObj(own)) return void bad.push("an eval links a Library group, or holds its own");
@@ -4454,14 +4584,7 @@ EVAL_TYPES.group = {
     // The group's own grader, or the lab's.
     const grader = isRef(own.grader) ? own.grader : isRef(ctx.grader) ? ctx.grader : null;
     if (gradedIn(own.every) && !grader) bad.push("a model-graded metric needs a grader: pick one on the eval, or make a connection the lab's grader");
-    // A grader is asked words, and needs a model to ask.
-    const p = grader && ctx.profiles ? ctx.profiles(grader.id) : null;
-    if (grader && ctx.profiles && !p) bad.push(`Connection ${grader.name || grader.id} not found`);
-    else if (p) {
-      const type = CONNECTION_TYPES[typeOf(p            )];
-      if (type && !(type.answers ?? ["prompt"]).includes("prompt")) bad.push(`the grader has to be asked words, and ${type.label} is not`);
-      else if (!type?.local && !String((p            ).model || "").trim()) bad.push("no model on the grader's connection");
-    }
+    graderProblems(grader, ctx, bad);
   },
   rules: t => ownList(t).map((m        , x        ) => ({
     key: `m${x}`, label: (isStr(m.metric) && m.metric) || METRICS[m.type]?.label || m.type,
@@ -4818,16 +4941,18 @@ STEP_TYPES.httpRequest = {
   label: "HTTP Request", slot: "target", asks: "request", verbatim: true, modelFrom: "step",
   description: "Sends a Power Automate step's own request, rebuilt from each record, with each target's words in it.",
   firstJobOnly: "an HTTP Request is sent from its item's record, so it is job 1's",
-  // A new target starts from the flow's own words and fields.
-  newStep: (step) => ({ type: "httpRequest", api: step.api, method: step.method, path: step.path,
+  // A new target starts from the flow's own words and fields, read by the
+  // job's HTTP API ([step] carries it: requestStepOf).
+  newStep: (step) => ({ type: "httpRequest", method: step.method, path: step.path,
                         query: clone(step.query), body: clone(step.body),
                         ...(HTTP_APIS[step.api] ?? HTTP_APIS.raw ).cellOf(step.body) }),
   in: "item", out: "text",
   apply: "runPipeline",
-  fields: ["type", "api", "method", "path", "query", "body", "prompt", "connection", "profile", "from", "system", "model", "settings", "prefill"],
+  fields: ["type", "method", "path", "query", "body", "prompt", "connection", "profile", "from", "system", "model", "settings", "prefill"],
+  // [step] carries its job's HTTP API, where the job has a flow step to say
+  // it (validatePipeline); the flow step's own check names an unknown one.
   validate(step, _ctx, bad, at){
     const api = HTTP_APIS[step.api];
-    if (!api) bad.push(`${at}: "${step.api}" is not an HTTP API this lab has`);
     if (!HTTP_METHODS.includes(step.method)) bad.push(`${at}: the method is one of ${HTTP_METHODS.join(", ")}`);
     if (!isStr(step.path) || !/^[/@]/.test(step.path)) bad.push(`${at}: the path starts with /`);
     if (!isObj(step.query) || !Object.values(step.query).every(isStr)) bad.push(`${at}: the query is names to text`);
@@ -4850,8 +4975,12 @@ STEP_TYPES.httpRequest = {
       }
     }
     if (isStr(step.prompt)) {
-      try { api.place(step.body, step        ); }
-      catch (e) { bad.push(`${at}: ${(e         ).message}`); }
+      // Built here as each call builds it, so an expression none of them
+      // could read is refused before the run rather than failing every call.
+      try {
+        const why = templateProblem([step.path, step.query, api.place(step.body, step        )]);
+        if (why) bad.push(`${at}: the request could not be built -- ${why}`);
+      } catch (e) { bad.push(`${at}: ${(e         ).message}`); }
     }
   },
 };
@@ -4876,7 +5005,6 @@ STEP_TYPES.echo = {
 // the flow reads one. No profile, no prompt -- the record is the reply.
 STEP_TYPES.recorded = {
   label: "Recorded", slot: "target", asks: "nothing", local: true, replaces: "recorded",
-  description: "Pre-recorded from previous workflow runs, replays only.",
   firstJobOnly: "a call's recorded reply is job 1's",
   in: "item", out: "text",
   apply: "runPipeline",
@@ -5104,7 +5232,7 @@ function requestStepOf(doc
 
 /** What httpRequestOf builds a request from: a target's template, and the
     loop its flow step sits in. */
-                                                                                                                          
+                                                                                                                               
 /** What httpReplyOf reads a reply with: the flow step, its API, its reading. */
                                                                                                      
 
@@ -5706,10 +5834,14 @@ function upgradePipeline   (doc   , ctx             = {})    {
     }
     return localSteps(ownBodiesNow(out), ctx)     ;
   }
-  if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15].includes(doc.version          )) return doc;
+  if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16].includes(doc.version          )) return doc;
+  // Every version reads as 16 first, then as 17: its requests read by their
+  // job's HTTP API (requestsOnJobApi).
+  const seventeen = (d           )    => localSteps(ownBodiesNow(requestsOnJobApi(d)), ctx)     ;
+  if (doc.version === 16) return seventeen(cutRefs(clone(doc)             ));
   // Every version reads as 15 first, then as 16: its Target profiles split
   // into AI Connections and Connection Profiles (answersFromProfiles).
-  const sixteen = (fifteen           )    => localSteps(ownBodiesNow(answersFromProfiles(fifteen, ctx)), ctx)     ;
+  const sixteen = (fifteen           )    => seventeen(answersFromProfiles(fifteen, ctx));
   if (doc.version === 15) return sixteen(cutRefs(clone(doc)             ));
   // Versions 13 and 14 read as 15 by migrating a job's cleaning modifier steps
   // and its List `preamble` setting into Pre-/Post-format filter sets
@@ -5854,6 +5986,37 @@ function localSteps(doc           , ctx            )            {
   return next;
 }
 
+/**
+ * Version 16 to 17 (#477): a request step has no HTTP API of its own; its
+ * job's flow step says it. A step that named another -- Any endpoint over an
+ * Anthropic flow, say -- keeps its body as it would have sent it, read again
+ * by the job's (its words and fields as that API holds them). A request in a
+ * job with no flow step, which could never run, is a Prompt keeping its words.
+ */
+function requestsOnJobApi(next           )            {
+  next.version = PIPELINE_VERSION;
+  const jobs            = Array.isArray(next.jobs) ? next.jobs : [];
+  const step = (st           , k        )            => {
+    if (!isObj(st) || st.type !== "httpRequest") return st;
+    const { api: own, ...rest } = st;
+    const flow = isObj(jobs[k]) ? flowStepOf(jobs[k]       ) : null;
+    if (!flow) {
+      return { type: "prompt", prompt: rest.prompt,
+               ...Object.fromEntries(["connection", "profile", "from"].filter(f => rest[f] != null).map(f => [f, rest[f]])) };
+    }
+    const was = HTTP_APIS[own          ], now = HTTP_APIS[flow.api];
+    if (!was || !now || own === flow.api) return rest;
+    let body = rest.body;
+    try { body = was.place(rest.body, rest        ); } catch { /* words that never placed: the template stands */ }
+    const { system: _s, model: _m, settings: _t, prefill: _p, ...kept } = rest;
+    return { ...kept, body, ...now.cellOf(body) };
+  };
+  if (Array.isArray(next.targets)) {
+    next.targets = next.targets.map((t           ) => (isObj(t) && Array.isArray(t.steps) ? { ...t, steps: t.steps.map(step) } : t));
+  }
+  return next;
+}
+
 /** [next] as version 9 held it, from any version before 10. */
 function nineOf(next           , ctx            )            {
   if (next.version === 9) { profileRefs(next); return next; }
@@ -5975,7 +6138,9 @@ function validatePipeline(input         , ctx          = {})           {
       if (entry.local && (st.profile != null || st.connection != null)) return void bad.push(`${where}: ${entry.label} asks no connection`);
       if (k > 0 && entry.firstJobOnly) return void bad.push(`${where}: ${entry.firstJobOnly}`);
       onlyFields(st, where, entry.fields ?? [], bad);
-      entry.validate(st, c, bad, where);
+      // A request is read and placed by its job's HTTP API.
+      const flow = entry.asks === "request" ? flowStepOf(doc.jobs[k]) : null;
+      entry.validate(flow ? { ...st, api: flow.api } : st, c, bad, where);
       answerRefsProblems(st, where, bad);
       if (st.from != null && !isRef(st.from)) bad.push(`${where} has to name the prompt it was picked from as { id, name }`);
       // A request is a flow step's, rebuilt from its records, and its
@@ -6031,6 +6196,10 @@ function validatePipeline(input         , ctx          = {})           {
     ? (c.sources?.(content.ref?.id)?.files ?? content.files ?? null) : null;
   const nonText = (contentFiles || []).map(f => String((f                             )?.name ?? f))
     .find(n => !/\.(?:txt|md|csv)$/i.test(n));
+  // A document goes whole to the API, so only a type that reads one is
+  // handed it -- never sent as an image or as text it is not.
+  const docType = (contentFiles || []).map(f => documentTypeOf(String((f                             )?.name ?? f)))
+    .find(Boolean) ?? null;
   targets.forEach((_, i) => {
     const conns = doc.jobs.map((_     , k        ) => (targetEntryOf(doc, i, k)?.local ? localConnOf(targetEntryOf(doc, i, k)) : lookup(targetAnswerOf(doc, i, k) )));
     conns.forEach((p         , k        ) => {
@@ -6050,6 +6219,19 @@ function validatePipeline(input         , ctx          = {})           {
       const type = CONNECTION_TYPES[typeOf(p            )];
       if (type && !(type.answers ?? ["prompt"]).includes(step.asks ?? "prompt")) {
         bad.push(`${n > 1 ? `${jobLabel(doc, k)}, ` : ""}${targetLabel(doc, i)}: ${step.label ?? "this step"} needs a profile that answers it, and ${type.label} does not`);
+        return;
+      }
+      // A request goes as the flow wrote it, so its connection has to speak
+      // the job's HTTP API; one the lab cannot tell (Any endpoint) passes.
+      const api = step.asks === "request" ? flowStepOf(doc.jobs[k])?.api : undefined;
+      const speaks = api && api !== "raw" && type?.speaks ? type.speaks(p            ) : null;
+      if (api && speaks && speaks !== api) {
+        const name = targetAnswerOf(doc, i, k)?.connection.name || type .label;
+        bad.push(`${n > 1 ? `${jobLabel(doc, k)}, ` : ""}${targetLabel(doc, i)}: ${name} can't send an ${HTTP_APIS[api]?.label ?? api} request`);
+      }
+      if (docType && type && !type.documents && sendsImage(doc.jobs[k])) {
+        const name = targetAnswerOf(doc, i, k)?.connection.name || type.label;
+        bad.push(`${n > 1 ? `${jobLabel(doc, k)}, ` : ""}${targetLabel(doc, i)}: ${name} can't read ${docType.label}s`);
       }
     });
     // The model is the profile's unless the target's step carries it.
@@ -6853,7 +7035,9 @@ function scenarioEvals(run             , i        , items
   return evals.map((t, j) => {
     const type = EVAL_TYPES[t.type];
     const whole = isWholeRun(type, t);
-    const base = { id: t.id, label: evalLabel({ evals }, j), whole, skipped: skipRest,
+    // An unnamed eval that links a group goes by the group's name, as the
+    // Evals Summary names it, not "Eval 1" (#519).
+    const base = { id: t.id, label: (t.name || "").trim() || t.group?.name || evalLabel({ evals }, j), whole, skipped: skipRest,
                    verdict: null                          , rules: type?.rules?.(t) ?? [], items: cells.map(() => null)                        ,
                    ran: 0, passed: 0, skippedItems: 0 };
     if (skipRest) {
@@ -6993,6 +7177,11 @@ function validateEvals(ev           , files
       if (scored.length > 1) say("expects its answer discarded, and states something the answer should hold too");
       continue;
     }
+    if (scored.some(m => m.type === "refused" && !m.not)) {
+      // Nor does a call that was refused: there is no answer.
+      if (scored.length > 1) say("expects its call refused, and states something the answer should hold too");
+      continue;
+    }
     const wants = scored.filter(m => !m.not && (m.type === "contains-all" || m.type === "contains-any"));
     const forbids = scored.filter(m => m.not && m.type === "contains");
     for (const m of wants) for (const t of values(m)) if (!matchable(t)) say(`expects ${t}, which its own scorer cannot match`);
@@ -7106,7 +7295,7 @@ export {
     loopReplyError, preparedSize,
     termIn, forbiddenIn, readCase, gradedSetFrom, caseItem, caseMetrics, metricLines, upgradeDatasetBody, takeAsExpected, readGroup, readGroupRun, datasetRules, upgradeResults, emptyTally, addToTally, isHostedUrl,
     SEED, REPLY_TOKENS_BEFORE, ANTHROPIC_MAX_TOKENS, asNumber, pinReplyTokens, mappingsFor,
-    CONNECTION_TYPES, HTTP_APIS, httpApiFor, httpRequestOf, httpReplyOf, readFlowReply, WdlError, localAnswer, typeOf, profileType, convertProfile, splitOllama, DECODING_KEYS, apiBase,
+    CONNECTION_TYPES, DOCUMENT_TYPES, documentTypeOf, documentUrl, attachmentOf, HTTP_APIS, httpApiFor, httpRequestOf, httpReplyOf, readFlowReply, WdlError, localAnswer, typeOf, listedConnection, profileType, convertProfile, splitOllama, DECODING_KEYS, apiBase,
     EDGE_448, budgetLabel, IMAGE_FORMATS, encoderQuality,
     tallyPercent, runPipeline, validateEvals, evalsWarnings, evalsJson, readList,
     scoreSingle, singleRules, parseCount, lengthHolds, countHolds, COUNT_OPS, LENGTH_OPS,
@@ -7119,8 +7308,8 @@ export {
 CONNECTION_FIELDS, SETTING_KEYS, OPTION_FIELDS, jobLabel, scenarioLabel, jobWords, versionProblem,
     contentOf, withContent, replyOf,
     targetsOf, targetLetter, targetLabel, targetStepOf, targetEntryOf, jobTargetType, targetAnswerOf, targetAnswerKey, recordedTargetOf, withTarget,
-    tokensOf, withTokens, sendsImage, withImage, flowStepOf, readAsOf, outOf, withOut, slotEntry, SLOTS,
-    filterSetsOf, withFilterSets, stageModifiers, beforeFilters,
+    tokensOf, withTokens, sendsImage, withImage, flowStepOf, requestStepOf, readAsOf, outOf, withOut, slotEntry, SLOTS,
+    filterSetsOf, withFilterSets, stageModifiers, beforeFilters, gradedText,
     blankPipeline, upgradePipeline, fatProfileRef, newId, newEval, newLink, validatePipeline, resolvePipeline, pipelineOfRun, stagesFor, profileIds, answerKeys,
     scenarioProfile, lastKind, lastModifier, modifierSummary, itemScores, scenarioEvals, scenarioPasses, groupPass, overallVerdict, passVerdict, isSkipped, failedScore,
     evalLabel, evalsOf, evalsDataset, casesRef, groupOf, groupOfMetrics, pipelineWarnings, itemsFound, previewOf, isWholeRun, EVAL_FIELDS,

package/lab/flows/wdl.mjs

@@ -363,6 +363,25 @@ function endOf(s        , from        )         {
   throw new WdlError("an @{ is never closed");
 }
 
+/** Why [template] cannot be evaluated whatever it is evaluated against -- an
+    expression in it that does not parse -- or null. Read before a run, so a
+    template every call would fail on is refused rather than sent. */
+export function templateProblem(template         )                {
+  try { walk(template); return null; }
+  catch (e) { if (e instanceof WdlError) return e.message; throw e; }
+  function walk(v         )       {
+    if (Array.isArray(v)) return v.forEach(walk);
+    if (isObj(v)) return Object.values(v).forEach(walk);
+    if (typeof v !== "string" || v.startsWith("@@")) return;
+    if (v.startsWith("@") && !v.startsWith("@{")) return void parse(v.slice(1));
+    for (let open = v.indexOf("@{"); open >= 0; open = v.indexOf("@{", open)) {
+      const close = endOf(v, open + 2);
+      parse(v.slice(open + 2, close));
+      open = close + 1;
+    }
+  }
+}
+
 /** [template] with every expression in it evaluated: each string that is
     one, or holds one, wherever it sits in objects and arrays. */
 export function evaluate(template         , ctx            )          {