evals-lab 0.12.0 → 0.14.0

package/lab/evals-core.mjs

@@ -34,7 +34,7 @@
 
 import yaml from "./js-yaml.mjs";
                                                                                                                                                  
-import { evaluate, asText, WdlError,                 } from "./flows/wdl.mjs";
+import { evaluate, asText, templateProblem, WdlError,                 } from "./flows/wdl.mjs";
 import { stepsOf as flowStepsOf, actionsByName, readerFor, readsOf,                          } from "./flows/record.mjs";
 import { flowApi,              } from "./flows/flowApi.mjs";
 import registerMetrics from "./metrics/builtin.mjs";
@@ -199,12 +199,23 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
                                                                                 
                                                                 
                  
+                                                                           
+                                                                         
+                                                                          
+                                                                           
+                           
                                                                             
                                                          
                    
                                 
  
 
+/** One kept response of a dynamic Source's item (#596). */
+                              
+               
+            
+ 
+
                               
                
                
@@ -347,17 +358,26 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
  * at. `filters` are tried in order -- each drop-item narrows the items the
  * next reads, and the first reject-reply that fires gates the answer.
  */
-/** Which side of Response Format a filter runs on (#405): `text` the raw reply before
-    it is read into items (rewrite-text / reject-text), `items` the items after
-    (drop-item, rewrite-item, keep-first, reject-reply). A filter's type decides
-    its stage; a set belongs to none (#432), and runs its share of filters in
-    each stage it is linked in. */
+/** Which side of Response Format a filter step sits on (#405): `text`
+    before it, on the raw reply (Pre-format), `items` after it (Post-format).
+    It also names what a filter reads: the reply's text (rewrite-text /
+    reject-text) or the items (drop-item, rewrite-item, keep-first,
+    reject-reply). A text filter runs at either stage, an item filter only
+    Post-format after a kind that reads items (#557, `stageReads`); a set
+    belongs to no stage (#432), and runs its share in each it is linked in. */
                                            
 
 /** The stage a filter type's scope belongs to: text filters read the raw
     reply; item, list and answer filters read the items it is read into. */
 export const stageOfScope = (scope        )              => (scope === "text" ? "text" : "items");
 
+/** What job's [stage] filters read when it answers as [kind] (#557):
+    Pre-format the text; Post-format the items after a kind that reads
+    items, and the text after any other. A filter runs there when its
+    `stageOfScope` is this. */
+export const stageReads = (stage             , kind                           )              =>
+  stage === "items" && OUTPUT_KINDS[kind ?? ""]?.offers?.includes("items") ? "items" : "text";
+
 /** A filter stage as the page names it: before the reply is formatted, or after. */
 export const FILTER_STAGE_LABEL                              = { text: "Pre-format", items: "Post-format" };
 
@@ -555,6 +575,10 @@ export const FILTER_STAGE_LABEL                              = { text: "Pre-form
                                                                            
                   
                     
+                                                                          
+                                                                          
+                                                                          
+                   
                          
                                                                                    
                                      
@@ -586,7 +610,15 @@ export const FILTER_STAGE_LABEL                              = { text: "Pre-form
                                                                         
                                                        
                 
+                                                                          
+                                                                       
+                  
+                      
  
+
+/** A reply the reply-token limit stopped: the limit the request sent, or
+    null where it sent none and the server's own applied. */
+                                                   
 
 /** The transport: a stage's prompt and image in, its reply out. */
                                                                               
@@ -607,6 +639,15 @@ export const FILTER_STAGE_LABEL                              = { text: "Pre-form
                                                                             
                                               
                 
+                                                                         
+                                                                         
+                                             
+                                           
+                                                          
+                  
+                                                                           
+                                                                              
+                      
  
 
 /** How much of one text a run keeps -- a reply's body, a grader's prompt or
@@ -672,6 +713,7 @@ export function kept(text        , max = KEPT_CHARS)         {
                
                      
                    
+                  
                         
                                     
  
@@ -897,6 +939,14 @@ export function kept(text        , max = KEPT_CHARS)         {
                     
                                                                                     
                        
+                                                                              
+                                                                               
+                                                                          
+                        
+                                                                          
+                                                                        
+                       
+                         
                                                                              
                                                                            
                  
@@ -1193,10 +1243,14 @@ const platformOf = (src
                                                                                 
                                                                          
                 
-                                                                            
-                                                                       
-                                                       
+                                                                         
+                                                                             
+                                                                   
                
+                                                                       
+                                                                        
+                                                                     
+                        
                                                                                
                                                 
                                       
@@ -1686,13 +1740,25 @@ function datasetRules(body         )                  {
                                           
                                                                                                        
                                                                        
+                                                                         
+                                                                           
+                                                   
+                                                                                   
                                                                                          
                                                                                                   
                                     
                           
+                                                                             
+                                                                             
+                                                            
+                      
                                                                            
                                  
                     
+                                                                             
+                                                                             
+                                                                               
+                                        
                                                                           
                                                                                
                                                                          
@@ -1993,6 +2059,35 @@ function asNumber(v         )                {
 // a tuned local engine is a llama.cpp profile with its settings.
 const CONNECTION_TYPES                                 = Object.create(null);
 
+// ---- documents (#558) ------------------------------------------------------
+// A file a model's API reads whole, sent unprocessed: the lab does not parse,
+// render or extract it. It travels where an image does -- the item's
+// attachment, a data URL -- carrying its name, which some APIs want beside
+// the bytes. Only a connection type marked `documents` is handed one.
+const DOCUMENT_TYPES                                                   = {
+  ".pdf": { media: "application/pdf", label: "PDF" },
+};
+
+/** The DOCUMENT_TYPES entry a file's name is, or null. */
+function documentTypeOf(name        )                                          {
+  const dot = String(name).lastIndexOf(".");
+  return dot < 0 ? null : DOCUMENT_TYPES[String(name).slice(dot).toLowerCase()] ?? null;
+}
+
+/** A document's bytes, base64, as the attachment a stage sends. */
+function documentUrl(name        , media        , base64        )         {
+  return `data:${media};name=${encodeURIComponent(name)};base64,${base64}`;
+}
+
+/** An attachment's media type, bytes and name (null for an image, which
+    carries none), or null for anything that is not a base64 data URL. */
+function attachmentOf(dataUrl        )                                                              {
+  const m = /^data:([^;,]+)((?:;[^;,=]+=[^;,]*)*);base64,(.*)$/s.exec(dataUrl);
+  if (!m) return null;
+  const name = /;name=([^;,]*)/.exec(m[2] )?.[1];
+  return { media: m[1] , data: m[3] , name: name == null ? null : decodeURIComponent(name) };
+}
+
 /** The key [key] as the header most endpoints expect, when there is one. */
 const bearer = (key                           )                         => (key ? { Authorization: "Bearer " + key } : {});
 
@@ -2058,6 +2153,7 @@ function ollamaType(id        , label        , cloud         )                 {
       const r = j                                                                   ;
       return { raw: r?.message?.content ?? "", finishReason: r?.done_reason ?? null };
     },
+    cutOff: { reason: "length", limit: (body) => asNumber((body.options                                       )?.num_predict) },
     listModels: (base, key) => listModels(base, cloud ? key : null),
     parseModels: idsFrom,
     ...(cloud ? { url: "https://ollama.com" } : { keyless: true }),
@@ -2105,6 +2201,13 @@ CONNECTION_TYPES.recorded = {
   request: sendsNothing, parseReply: sendsNothing, listModels: sendsNothing, parseModels: () => [],
 };
 
+/** Whether Setup and the connection pickers list a stored connection. A local
+    type's (Echo, Recorded) is a target step's stand-in now (`localSteps`), so
+    a row of one is kept only for the pipelines and runs that name it. */
+function listedConnection(conn                           )          {
+  return !CONNECTION_TYPES[typeOf(conn                               )]?.local;
+}
+
 /** A local type's answer to one stage, as a model's would come back. An item
     with no text of its own (Prompt only) is answered from the prompt, and
     that reply is read against nothing -- the prompt is the reply, not an
@@ -2120,9 +2223,19 @@ function localAnswer(conn          , text               , sent        , record
 const chatSchema = (body                         , schema                         )                          =>
   ({ ...body, response_format: { type: "json_schema", json_schema: { name: "reply", strict: true, schema } } });
 
+/** An item's attachment as a chat completions content part: an image as
+    image_url, a document as a file part, named, its bytes a data URL. */
+function chatAttachment(dataUrl        )         {
+  const a = attachmentOf(dataUrl);
+  if (!a || a.media.startsWith("image/")) return { type: "image_url", image_url: { url: dataUrl } };
+  return { type: "file", file: { filename: a.name ?? "document", file_data: `data:${a.media};base64,${a.data}` } };
+}
+
 CONNECTION_TYPES["openai-compatible"] = {
   id: "openai-compatible", label: "OpenAI-compatible",
   description: "OpenAI, OpenRouter, vLLM, LM Studio: any chat completions endpoint.",
+  // A hosted provider wants a key; a server of your own may not.
+  keyOptional: (conn) => !isHostedUrl(String(conn.url ?? "")),
   settings: [
     // A reasoning model refuses a temperature.
     { key: "temperature", label: "Temperature", control: "text", default: "", hint: "",
@@ -2145,13 +2258,17 @@ CONNECTION_TYPES["openai-compatible"] = {
         ...(predict == null ? {} : hosted ? { max_completion_tokens: predict } : { max_tokens: predict }),
         messages: [{ role: "user", content: [
           { type: "text", text: prompt },
-          ...(dataUrl ? [{ type: "image_url", image_url: { url: dataUrl } }] : []),
+          ...(dataUrl ? [chatAttachment(dataUrl)] : []),
         ] }],
       },
     };
   },
+  // OpenAI's file part; a compatible server that has none refuses the call,
+  // and the refusal is that item's error.
+  documents: true,
   structured: chatSchema,
   parseReply: chatReply,
+  cutOff: { reason: "length", limit: (body) => asNumber(body.max_completion_tokens ?? body.max_tokens) },
   listModels, parseModels: idsFrom,
 };
 
@@ -2174,15 +2291,15 @@ CONNECTION_TYPES.anthropic = {
   answers: ["prompt", "request"],
   speaks: () => "anthropic",
   // The Messages API: the x-api-key and anthropic-version headers, base64
-  // image content blocks, and /v1/models for the list.
+  // image and document content blocks, and /v1/models for the list.
+  documents: true,
   request(conn, prompt, dataUrl, base, key) {
     const temp = asNumber(conn.temperature);
     const predict = asNumber(conn.maxTokens) ?? ANTHROPIC_MAX_TOKENS;
     const content           = [];
-    if (dataUrl) {
-      const m = /^data:(image\/[^;]+);base64,(.*)$/s.exec(dataUrl);
-      if (m) content.push({ type: "image", source: { type: "base64", media_type: m[1], data: m[2] } });
-    }
+    const a = dataUrl ? attachmentOf(dataUrl) : null;
+    if (a?.media.startsWith("image/")) content.push({ type: "image", source: { type: "base64", media_type: a.media, data: a.data } });
+    else if (a) content.push({ type: "document", source: { type: "base64", media_type: a.media, data: a.data } });
     content.push({ type: "text", text: prompt });
     return {
       url: base + "/messages",
@@ -2203,6 +2320,7 @@ CONNECTION_TYPES.anthropic = {
     const raw = (r?.content || []).filter(b => b.type === "text").map(b => b.text).join("");
     return { raw, finishReason: r?.stop_reason ?? null };
   },
+  cutOff: { reason: "max_tokens", limit: (body) => asNumber(body.max_tokens) },
   // A flow's request, keyed as the Messages API is -- what the HTTP endpoint
   // type's Anthropic template sets. [base] is the /v1 base words hang off;
   // the step's path is the whole path production called, so it goes to the
@@ -2227,6 +2345,7 @@ const SPEED_IMAGE = "before", SPEED_CACHE = "off", SPEED_THINKING = "off";
 CONNECTION_TYPES["llama.cpp"] = {
   id: "llama.cpp", label: "llama.cpp",
   description: "A llama-server you run, with its sampler settings.",
+  keyOptional: () => true,
   settings: [
     { key: "temperature", label: "Temperature", control: "text", default: "", hint: "" },
     { key: "seed", label: "Seed", control: "text", default: "", hint: "" },
@@ -2285,6 +2404,7 @@ CONNECTION_TYPES["llama.cpp"] = {
   // llama-server takes OpenAI's response_format, a schema and all.
   structured: chatSchema,
   parseReply: chatReply,
+  cutOff: { reason: "length", limit: (body) => asNumber(body.n_predict) },
   listModels, parseModels: idsFrom,
   // A reply that arrived is still a failure when this profile asked for the
   // Speed looping check (today's speedReplyError, now an opt-in setting).
@@ -2511,6 +2631,7 @@ CONNECTION_TYPES.http = {
                 ...Object.entries(HTTP_APIS).map(([value, a]) => ({ value, label: a.label }))] },
   ],
   answers: ["request"],
+  keyOptional: (conn) => conn.auth === "none",
   speaks(conn) {
     const api = String(conn.requestFormat || "") || httpApiFor(originBase(conn.url));
     return HTTP_APIS[api] && api !== "raw" ? api : null;
@@ -2658,6 +2779,18 @@ function connectionReply(conn          , j         )
 }
 
 /** Why a reply [conn] received is a failure though it arrived, or null. */
+/** How a reply finished, as a stage keeps it (#597): the reason [conn]'s
+    API gave, and -- where that reason is its type's reply-token limit -- the
+    limit [body], the request sent, carried. A type that says no reason
+    keeps nothing. */
+function replyFinish(conn          , body         , finishReason               )                                    {
+  if (!finishReason) return {};
+  const cut = typeEntry(conn).cutOff;
+  if (!cut || finishReason !== cut.reason) return { finish: finishReason };
+  const limit = body && typeof body === "object" ? cut.limit(body                           ) : null;
+  return { finish: finishReason, stopped: { limit } };
+}
+
 function connectionReplyProblem(conn          , raw        , finishReason               )                {
   return typeEntry(conn).replyProblem?.(conn, raw, finishReason) ?? null;
 }
@@ -2797,7 +2930,7 @@ function readCase(kase      , res                                   , plain = fa
     }
     return r ? [r                 ] : [];
   });
-  const s = scoreOf(metrics) ?? { pass: true, score: null, metrics };
+  const s = erroredScore(scoreOf(metrics), res.error) ?? { pass: !res.error, score: null, metrics };
   const counted = metrics.filter(r => r.weight !== 0);
   return { ...s, metrics, found: s.found ?? [], missed: s.missed ?? [], invented: s.invented ?? [],
            discarded: !!res.error, count: res.error ? 0 : (res.terms || []).length,
@@ -3183,7 +3316,8 @@ async function runPipeline(stages                  , dataUrl               , cal
     total += res.ms;
     transcript.push({ n: i + 1, kind, conn: res.conn ?? null, withImage, sent: res.sent ?? sent,
                       got: res.raw ?? "", ms: res.ms, error: res.error, ...(res.said != null ? { said: res.said } : {}),
-                      ...(res.body != null ? { body: kept(res.body) } : {}) });
+                      ...(res.body != null ? { body: kept(res.body) } : {}),
+                      ...(res.finish ? { finish: res.finish } : {}), ...(res.stopped ? { stopped: res.stopped } : {}) });
     if (res.error) return result({ error: named(res.error) });
     // A stage that failed is the end of the job: nothing after it is asked.
     // Handed nothing, the next stage still answers -- about an empty list --
@@ -3195,12 +3329,23 @@ async function runPipeline(stages                  , dataUrl               , cal
     const pre = applyFilterSet(st.before || [], { body: res.raw ?? "", items: [], instruction: against }, "text");
     const by = (filter        , stage             , rejected         )             => ({ filter, stage, job: i + 1, rejected });
     if (pre.reject != null) return result({ error: named(`discarded: ${pre.reject}`), filteredBy: by(pre.reject, "text", true) });
+    if (pre.rewrote) transcript[transcript.length - 1] .filtered = { got: pre.body, by: pre.rewrote };
     const read = reader.read(pre.body, { echo: against, parsing: opts.parsing, settings: st.settings });
     if (read.error) return result({ error: named(read.error) });
     const mod = applyModifiers(mods[i], kind, read.value, { instruction: against, body: read.body ?? res.raw ?? "" });
     if (mod.reject) {
       return result({ error: named(`discarded: ${mod.reject}`), ...(mod.by ? { filteredBy: by(mod.by, "items", true) } : {}) });
     }
+    // A kind that reads no items runs its Post-format filters on its text,
+    // once read (#557); what they leave is what is graded.
+    if (st.after?.length && typeof mod.value === "string") {
+      const post = applyFilterSet(st.after, { body: mod.value, items: [], instruction: against }, "text");
+      if (post.reject != null) return result({ error: named(`discarded: ${post.reject}`), filteredBy: by(post.reject, "items", true) });
+      if (post.rewrote) {
+        transcript[transcript.length - 1] .filtered = { got: post.body, by: [...(pre.rewrote || []), ...post.rewrote] };
+        mod.value = post.body;
+      }
+    }
     // A filter that dropped the last item: what a zero result is explained by.
     const emptied = mod.by && reader.terms && !reader.terms(mod.value).length ? { filteredBy: by(mod.by, "items", false) } : {};
     const nothing = handsOn ? reader.handsOn?.(mod.value) : null;
@@ -3209,7 +3354,9 @@ async function runPipeline(stages                  , dataUrl               , cal
     held = { kind, value };
     const gone = [...(read.dropped || []), ...mod.dropped];
     echoed = (read.echoed || 0) + mod.dropped.filter(d => d.by === ECHO).length;
-    done.push({ reply: res.raw ?? "", rendered: reader.render?.(value) || {} });
+    // {reply} and {stageN} hand on the text as its filters left it -- what
+    // was graded -- never the raw reply a filter cleaned (#557).
+    done.push({ reply: transcript[transcript.length - 1] .filtered?.got ?? res.raw ?? "", rendered: reader.render?.(value) || {} });
     if (!handsOn) {
       dropped = gone;
       return result({ echoed, dropped, ...emptied });
@@ -3309,7 +3456,7 @@ function applyModifiers   (list                                 , kind        ,
 // 16: a target names an AI Connection and, optionally, a Connection Profile,
 // where it named one Target profile; a run's table is keyed by the pair
 // (answerKey) -- the Target profile split in the store as it opens (#445).
-const PIPELINE_VERSION = 17         ;
+const PIPELINE_VERSION = 18         ;
 
 // Plain objects, so an entry is added by assignment and a reader never needs
 // to know which registered it.
@@ -3591,9 +3738,10 @@ function splitProfile(p         )
 }
 
 /** [conn] with [profile]'s settings over it: the one set of settings a
-    request is built from, the shape a Target profile had. A profile sets
-    no connection field. */
-function mergeAnswer(conn                        , profile                           )          {
+    request is built from, the shape a Target profile had. A profile -- or
+    (Default)'s settings, which are one without its id and name -- sets no
+    connection field. */
+function mergeAnswer(conn                        , profile                                      )          {
   const out          = { ...conn };
   const scoped = connectionScoped(typeOf(conn            ));
   for (const [k, v] of Object.entries(profile ?? {})) {
@@ -3610,7 +3758,7 @@ function mergeAnswer(conn                        , profile
  
 
 /** The key a run's table holds [connection] under, with [profile] -- or with
-    none, the lab's defaults. A grader names its connection bare, its id alone,
+    none, (Default)'s settings. A grader names its connection bare, its id alone,
     so it never meets a target's pair (see `answersOf`). */
 const answerKey = (connection        , profile                )         => `${connection}.${profile ?? ""}`;
 
@@ -3645,9 +3793,13 @@ function targetAnswerKey(doc
  * answer key resolved to the settings it sends -- the profile's connection
  * where it names one, else the key's, with the profile's settings over it.
  * A grader's bare id resolves as the Target profile it named did: its
- * connection under the profile split from it. Null where any part is gone.
+ * connection under the profile split from it. One that names no profile, or
+ * a grader with none split beside it, takes [fallback] instead -- (Default)'s
+ * settings, which sit under no named profile, so a profile is still what it
+ * sets (Setup's Used by counts the same, `defaultUses`). Null where any part
+ * is gone.
  */
-function answersOf(connections                         , profiles                              ) {
+function answersOf(connections                         , profiles                              , fallback                  ) {
   const connection = (id        ) => connections.find(c => c?.id === id) ?? null;
   const profile = (id        ) => profiles.find(p => p?.id === id) ?? null;
   const split = (id        )             => {
@@ -3660,7 +3812,7 @@ function answersOf(connections                         , profiles
     const p = pid ? profile(pid) : null;
     if (pid && !p) return null;
     const c = connection(isRef(p?.connection) ? p .connection .id : at.connection);
-    return c ? mergeAnswer(c, p) : null;
+    return c ? mergeAnswer(c, p ?? fallback) : null;
   };
   return { connection, profile, split, resolve };
 }
@@ -3731,16 +3883,17 @@ function onlyFields(obj        , at        , allowed                   , bad
 
 // A Source is referenced, with the two numbers that decide how many items a
 // run over it is: the first N files, and how many times over.
-// A folder of files: images sent as images, text files as {text}. The
-// extensions are server.py's FILE_TYPES, which the server holds uploads to.
+// A folder of files: images sent as images, documents whole (DOCUMENT_TYPES),
+// text files as {text}. The extensions are server.py's FILE_TYPES, which the
+// server holds uploads to.
 const FILE_LIBRARY_TEXT = [".txt", ".md", ".csv"]         ;
 SOURCE_TYPES.files = {
   label: "File Library",
   icon: "folder",
-  description: "A folder of images and text files you upload.",
+  description: "A folder of images, PDFs and text files you upload.",
   group: "Files",
   noun: "file",
-  uploads: [".jpg", ".jpeg", ".png", ".heic", ".heif", ".dng", ".tif", ".tiff", ...FILE_LIBRARY_TEXT],
+  uploads: [".jpg", ".jpeg", ".png", ".heic", ".heif", ".dng", ".tif", ".tiff", ...Object.keys(DOCUMENT_TYPES), ...FILE_LIBRARY_TEXT],
   sendsImage: true,
   hasActions: false,
   itemText: name => FILE_LIBRARY_TEXT.some(ext => name.toLowerCase().endsWith(ext)),
@@ -3757,7 +3910,9 @@ SOURCE_TYPES.workflow = {
   icon: "workflow",
   description: "A cloud flow, with records of its steps' calls from its run history.",
   group: "Workflows",
-  noun: "record",
+  // Each item reads as a call, as the Source's Calls group names them, though
+  // the file holding one is a record.
+  noun: "call",
   uploads: [".json"],
   sendsImage: false,
   hasActions: true,
@@ -3810,15 +3965,15 @@ SOURCE_TYPES.dynamic = {
 };
 
 // HTTP GET (docs/dynamic-sources.md § 1): the safest pull provider. It fetches
-// a URL through the relay server-side -- Snapshot now freezes one Call into the
-// Source, Live produces one fresh at submit -- so the page drives only the
+// a URL through the relay server-side -- Fetch now keeps a response, Live
+// data fetches it again at submit -- so the page drives only the
 // create dialog and the Call labels; the fetch, redaction and token-refusal are
 // the server's (its SOURCE_PROVIDERS mirror in server.py).
 DYNAMIC_PROVIDERS["http-get"] = {   // provider: the HTTP GET adapter's own id
   id: "http-get",                   // provider: its id, the row's config.provider
   label: "HTTP GET",
   icon: "square-code",
-  noun: "call",
+  noun: "response",
   tier: "pull",
   config: ["url"],
   configProblems(config){
@@ -3846,8 +4001,8 @@ const API_METHODS = ["GET", "POST", "PUT", "PATCH", "DELETE"];
 DYNAMIC_PROVIDERS["api"] = {      // provider: the API call adapter's own id
   id: "api",                      // provider: its id, the row's config.provider
   label: "API call",
-  icon: "square-code",
-  noun: "call",
+  icon: "arrow-left-right",
+  noun: "response",
   tier: "pull",
   config: ["method", "url", "reqHeaders", "body", "connection"],
   configProblems(config){
@@ -3876,7 +4031,7 @@ DYNAMIC_PROVIDERS["api"] = {      // provider: the API call adapter's own id
 // or an Upload… by hand, formalised as a provider. The lab fetches and executes
 // nothing: a Supplied item is named by the id the caller gives it (its file
 // name), which is what joins it to a case. It has no config and no server-side
-// production, so no Snapshot now and no live; its Calls are the items handed in,
+// production, so no Fetch now and no live data; its Calls are the items handed in,
 // redacted on the way in like any other (server.py's take_dynamic). A hosted run
 // grades those Calls; a solely-Supplied Source with none is run only via the CLI.
 DYNAMIC_PROVIDERS["supplied"] = {   // provider: the Supplied adapter's own id
@@ -3906,6 +4061,7 @@ DYNAMIC_PROVIDERS["file"] = {   // provider: the File on disk adapter's own id
   label: "File on disk",
   icon: "file",
   noun: "file",
+  produceLabel: "Read now",
   tier: "local",
   config: ["path"],
   configProblems(config){
@@ -3929,6 +4085,7 @@ DYNAMIC_PROVIDERS["command"] = {   // provider: the Command adapter's own id
   label: "Command",
   icon: "terminal",
   noun: "call",
+  produceLabel: "Run now",
   tier: "local",
   config: ["command", "args"],
   configProblems(config){
@@ -4011,11 +4168,14 @@ CONTENT_TYPES.source = {
   description: "Every file or record in a Source from the Library.",
   // `live` is a dynamic Source's per-run snapshot-vs-live override
   // (docs/dynamic-sources.md); meaningless for another kind, ignored there.
-  fields: ["type", "ref", "first", "loops", "live"],
+  fields: ["type", "ref", "first", "loops", "live", "pin"],
   runFields: ["files", "revs"],
   validate(c, ctx, bad){
     if (!isRef(c.ref)) return void bad.push("a Source content has to name its Source as { id, name }");
     if (c.live != null && typeof c.live !== "boolean") bad.push("a Source content's live is true or false");
+    if (c.pin != null && !(isObj(c.pin) && isStr(c.pin.name) && c.pin.name && Number.isInteger(c.pin.n) && c.pin.n >= 0)) {
+      bad.push("a Source content's pin names a response as { name, n }");
+    }
     if (c.first != null && !(Number.isInteger(c.first) && c.first > 0)) {
       bad.push("Run first wants a positive number, or empty for all");
     }
@@ -4116,7 +4276,7 @@ LEGACY_TESTS.single = {
     const raws           = [], terms           = [];
     for (const r of ress) {
       if (!r) continue;
-      raws.push(r.transcript?.[r.transcript.length - 1]?.got ?? r.raw ?? "");
+      raws.push(gradedText(r));
       terms.push(...(r.terms || []));
     }
     const s = scoreSingle(t, terms, raws, !(kind != null && OUTPUT_KINDS[kind]?.terms));
@@ -4181,11 +4341,32 @@ function metricsProblems(list         , at        , bad          )       {
 /** Whether any of [list] asks a grader. */
 const gradedIn = (list         )          => Array.isArray(list) && list.some((m           ) => !!METRICS[m?.type]?.graded);
 
+/** Whether [error] is a reply the job's rules threw away, as the Discarded
+    metric reads it: a result that came back, where any other error is a
+    request that never did. */
+const thrownAway = (error                           )          => !!error && error.startsWith("discarded: ");
+
+/** [s] as an errored result's score: whatever its metrics read, a result
+    with no reply fails, and only a discarded reply is left to them (#544) --
+    or a refused call, to a case that scores whether it was (#559). */
+function erroredScore                              (s   , error                           )    {
+  const asksRefusal = !!s?.metrics?.some(r => r.weight !== 0 && METRICS[r.type]?.readsRefusal);
+  return s && error != null && !thrownAway(error) && !asksRefusal ? { ...s, pass: false } : s;
+}
+
+/** The reply a metric grades and Results shows (#546): the last stage's
+    reply after its filters (#557), or the raw reply. */
+function gradedText(res                                                      )         {
+  const last = res?.transcript?.at(-1);
+  return last?.filtered?.got ?? last?.got ?? res?.raw ?? "";
+}
+
 /** What a metric reads of one result. */
 function metricInput(res                    , kase             , more          )              {
   const last = res.transcript?.at(-1);
-  const text = last?.got ?? res.raw ?? "";
-  return { text, said: last?.said ?? null, terms: res.terms ?? [], replies: [text], plain: !!more.plain,
+  const text = gradedText(res);
+  // `of: "said"` reads the reply as it came back, before any filter.
+  return { text, said: last?.said ?? (last?.filtered ? last.got ?? null : null), terms: res.terms ?? [], replies: [text], plain: !!more.plain,
            asked: last?.sent ?? null,
            // A case that has been given its own expected reply ("Take B as
            // expected") is compared with that; otherwise the Source's recorded
@@ -4201,7 +4382,7 @@ function runInput(ress                                       , plain         )
   let ms = 0;
   for (const r of ress) {
     if (!r) continue;
-    replies.push(r.transcript?.[r.transcript.length - 1]?.got ?? r.raw ?? "");
+    replies.push(gradedText(r));
     terms.push(...(r.terms || []));
     ms += r.ms ?? 0;
   }
@@ -4217,6 +4398,19 @@ function readMetric(m        , input             , ctx           )
   const label = (isStr(m.metric) && m.metric.trim()) || entry?.label || m.type;
   if (!entry) return { type: m.type, label, weight, pass: false, score: 0, reason: `"${m.type}" is not a metric this lab has` };
   const read = m.of === "said" ? { ...input, text: input.said ?? input.text } : input;
+  // An errored result has no reply to read (#544): a metric that reads one
+  // fails on the error, and a grader is never asked about the empty reply.
+  if (input.error != null && !(entry.readsResult && (thrownAway(input.error) || entry.readsRefusal))) {
+    const fail                = { type: m.type, label, weight, pass: false, score: 0, reason: input.error };
+    if (entry.graded) return fail;
+    // What it would have missed still counts toward a run's totals.
+    let o                                   = null;
+    try { o = entry.score(read, m, {}); } catch { o = null; }
+    if (!o) return null;
+    if (typeof (o                    ).then === "function") return fail;
+    const { found, missed, invented } = o                 ;
+    return { ...fail, ...(found ? { found } : {}), ...(missed ? { missed } : {}), ...(invented ? { invented } : {}) };
+  }
   const done = (o                      )                       => {
     if (!o) return null;
     const turned = m.not ? { ...o, pass: !o.pass, score: entry.counts ? -o.score : 1 - o.score, reason: `not: ${o.reason}` } : o;
@@ -4249,6 +4443,18 @@ function readMetric(m        , input             , ctx           )
   } catch (e) { return timed(failed(e)); }
 }
 
+/** What stops [grader] being asked, where the lookups can say: a grader is
+    asked words, and needs a model to ask. */
+function graderProblems(grader            , ctx               , bad          )       {
+  const p = grader && ctx.profiles ? ctx.profiles(grader.id) : null;
+  if (grader && ctx.profiles && !p) bad.push(`Grader ${grader.name || grader.id} not found`);
+  else if (p) {
+    const type = CONNECTION_TYPES[typeOf(p            )];
+    if (type && !(type.answers ?? ["prompt"]).includes("prompt")) bad.push(`the grader has to be asked words, and ${type.label} is not`);
+    else if (!type?.local && !String((p            ).model || "").trim()) bad.push("no model on the grader's connection");
+  }
+}
+
 /** An eval's score from its metrics' readings, the way [mode] says: all must
     pass, or weighted points against [threshold]. What a case metric found,
     missed and invented is carried up, for a run's totals. Null for none. */
@@ -4277,7 +4483,7 @@ async function readMetrics(list                   , input             , ctx
     const r = await readMetric(m, input, ctx);
     if (r) metrics.push(r);
   }
-  return scoreOf(metrics, mode, threshold);
+  return erroredScore(scoreOf(metrics, mode, threshold), input.error);
 }
 
 /** [list] read over a whole run, synchronously: each metric over the run's
@@ -4331,14 +4537,7 @@ LEGACY_TESTS.metrics = {
     // The eval's own grader, or the lab's.
     const grader = isRef(t.grader) ? t.grader : isRef(ctx.grader) ? ctx.grader : null;
     if (gradedIn(t.metrics) && !grader) bad.push("a model-graded metric needs a grader: pick one on the eval, or make a connection the lab's grader");
-    // A grader is asked words, and needs a model to ask.
-    const p = grader && ctx.profiles ? ctx.profiles(grader.id) : null;
-    if (grader && ctx.profiles && !p) bad.push(`Connection ${grader.name || grader.id} not found`);
-    else if (p) {
-      const type = CONNECTION_TYPES[typeOf(p            )];
-      if (type && !(type.answers ?? ["prompt"]).includes("prompt")) bad.push(`the grader has to be asked words, and ${type.label} is not`);
-      else if (!type?.local && !String((p            ).model || "").trim()) bad.push("no model on the grader's connection");
-    }
+    graderProblems(grader, ctx, bad);
   },
   rules: t => (Array.isArray(t.metrics) ? t.metrics : []).map((m        , x        ) => ({
     key: `m${x}`, label: (isStr(m.metric) && m.metric) || METRICS[m.type]?.label || m.type,
@@ -4450,6 +4649,11 @@ EVAL_TYPES.group = {
       if (held && t.pin != null && typeof held.versions === "number" && t.pin > held.versions) {
         bad.push(`${g.name || g.id} has no version ${t.pin}`);
       }
+      // The grader the group asks: the run's copy of the link carries it, a
+      // pipeline's reads it from the group as the lab lists it. One that is
+      // gone would fail every item on a metric that cannot run (#545).
+      const grader = isRef(t.grader) ? t.grader : isRef(held?.grader) ? held .grader        : null;
+      graderProblems(grader, ctx, bad);
       return;
     }
     if (!isObj(own)) return void bad.push("an eval links a Library group, or holds its own");
@@ -4466,14 +4670,7 @@ EVAL_TYPES.group = {
     // The group's own grader, or the lab's.
     const grader = isRef(own.grader) ? own.grader : isRef(ctx.grader) ? ctx.grader : null;
     if (gradedIn(own.every) && !grader) bad.push("a model-graded metric needs a grader: pick one on the eval, or make a connection the lab's grader");
-    // A grader is asked words, and needs a model to ask.
-    const p = grader && ctx.profiles ? ctx.profiles(grader.id) : null;
-    if (grader && ctx.profiles && !p) bad.push(`Connection ${grader.name || grader.id} not found`);
-    else if (p) {
-      const type = CONNECTION_TYPES[typeOf(p            )];
-      if (type && !(type.answers ?? ["prompt"]).includes("prompt")) bad.push(`the grader has to be asked words, and ${type.label} is not`);
-      else if (!type?.local && !String((p            ).model || "").trim()) bad.push("no model on the grader's connection");
-    }
+    graderProblems(grader, ctx, bad);
   },
   rules: t => ownList(t).map((m        , x        ) => ({
     key: `m${x}`, label: (isStr(m.metric) && m.metric) || METRICS[m.type]?.label || m.type,
@@ -4864,8 +5061,12 @@ STEP_TYPES.httpRequest = {
       }
     }
     if (isStr(step.prompt)) {
-      try { api.place(step.body, step        ); }
-      catch (e) { bad.push(`${at}: ${(e         ).message}`); }
+      // Built here as each call builds it, so an expression none of them
+      // could read is refused before the run rather than failing every call.
+      try {
+        const why = templateProblem([step.path, step.query, api.place(step.body, step        )]);
+        if (why) bad.push(`${at}: the request could not be built -- ${why}`);
+      } catch (e) { bad.push(`${at}: ${(e         ).message}`); }
     }
   },
 };
@@ -4890,7 +5091,6 @@ STEP_TYPES.echo = {
 // the flow reads one. No profile, no prompt -- the record is the reply.
 STEP_TYPES.recorded = {
   label: "Recorded", slot: "target", asks: "nothing", local: true, replaces: "recorded",
-  description: "Pre-recorded from previous workflow runs, replays only.",
   firstJobOnly: "a call's recorded reply is job 1's",
   in: "item", out: "text",
   apply: "runPipeline",
@@ -5033,6 +5233,22 @@ function withContent                           (doc   , content                )
   return { ...doc, jobs: [withStep(first, "attachContent", content ? { type: "attachContent", content } : null), ...rest] };
 }
 
+/**
+ * Whether target [i]'s words in job [k] have the item's text joined after
+ * them (#597): job 1, a target that asks a prompt, text content -- a Source
+ * of text items, or inline text, as its CONTENT_TYPES entry says -- and no
+ * {text} in the words to place it (textPrompt). A later job's reply goes
+ * only where a token places it, and an image or a document is sent as a part
+ * of its own, so neither is joined. [ctx] reads a Source as validation does.
+ */
+function contentJoined(doc                                                                                  , i        , k        ,
+                       ctx                = {})          {
+  if (k !== 0 || targetEntryOf(doc, i, k)?.asks !== "prompt") return false;
+  if (String(targetStepOf(doc, i, k)?.prompt ?? "").includes("{text}")) return false;
+  const content = contentOf(doc);
+  return !!content && (CONTENT_TYPES[content.type]?.text?.(content, ctx) ?? null) != null;
+}
+
 /** A document's targets, in column order. */
 function targetsOf(doc                                                           )           {
   return Array.isArray(doc?.targets) ? doc .targets             : [];
@@ -5252,6 +5468,15 @@ function stageModifiers(job         , resolve
   return out;
 }
 
+/** A job's Post-format filters for a run when its kind reads no items
+    (#557), in document order: every link's on its `items` filters step, which
+    a run applies to the text after the kind reads it. A kind that reads items
+    has none here -- `stageModifiers` runs its Post-format links. */
+function afterFilters(job         , resolve                                                          )           {
+  if (stageReads("items", outOf(job)?.kind) !== "text") return [];
+  return (filterStepOf(job, "items")?.filterSets || []).flatMap(link => linkFilters(link, resolve));
+}
+
 /** A job's Pre-format (text) filters for a run (#405), in document order:
     the filters of every link on its `text` filters step, which a run applies to
     the raw reply before the kind reads it into items. [resolve] gives a Library
@@ -5419,6 +5644,12 @@ function versionProblem(doc         )                {
                                                                             
                                              
                                                            
+                                                                           
+                                                                           
+                                                                          
+                                                                         
+                           
+                                                                                           
  
 
 /**
@@ -5710,7 +5941,8 @@ function ownBodiesNow(doc           )            {
 
 function upgradePipeline   (doc   , ctx             = {})    {
   // A current document is read as it is, but for a profile reference the Runs
-  // tab saved whole (see profileRefs), which is cut back, and a step on a
+  // tab saved whole (see profileRefs), which is cut back, a Read Reply's
+  // leftover `preamble: false` (withoutLeftoverPreamble), and a step on a
   // profile a target step stands in for (localSteps).
   if (isObj(doc) && doc.version === PIPELINE_VERSION) {
     let out = doc             ;
@@ -5718,12 +5950,16 @@ function upgradePipeline   (doc   , ctx             = {})    {
       out = clone(out)             ;
       profileRefs(out);
     }
-    return localSteps(ownBodiesNow(out), ctx)     ;
+    return localSteps(ownBodiesNow(withoutLeftoverPreamble(out)), ctx)     ;
   }
-  if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16].includes(doc.version          )) return doc;
+  if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17].includes(doc.version          )) return doc;
+  // Every version reads as 17 first, then as 18: a saved-mode Source content
+  // pins one response (savedPinned).
+  const eighteen = (d           )    => localSteps(ownBodiesNow(savedPinned(d, ctx)), ctx)     ;
+  if (doc.version === 17) return eighteen(cutRefs(clone(doc)             ));
   // Every version reads as 16 first, then as 17: its requests read by their
   // job's HTTP API (requestsOnJobApi).
-  const seventeen = (d           )    => localSteps(ownBodiesNow(requestsOnJobApi(d)), ctx)     ;
+  const seventeen = (d           )    => eighteen(requestsOnJobApi(d));
   if (doc.version === 16) return seventeen(cutRefs(clone(doc)             ));
   // Every version reads as 15 first, then as 16: its Target profiles split
   // into AI Connections and Connection Profiles (answersFromProfiles).
@@ -5769,11 +6005,13 @@ function filtersFromCleaning(next           )            {
   const isFilters = (st           ) => isObj(st) && st.type === "filters";
   next.jobs = next.jobs.map((job           ) => {
     if (!isObj(job) || !Array.isArray(job.steps)) return job;
-    const steps = job.steps               ;
-    const readReply = steps.find(st => isObj(st) && st.type === "readReply")                         ;
+    const readReply = job.steps.find((st           ) => isObj(st) && st.type === "readReply")                         ;
     const preamble = !!(readReply && isObj(readReply.out) && readReply.out.preamble === true);
+    // Stripped from every Read Reply, `false` included, before the job is
+    // asked whether it has anything to migrate (#604).
+    const steps = (job.steps               ).map(withoutPreamble);
     const stale = steps.some(st => isFilters(st) && (st             ).stage == null);
-    if (!steps.some(isCleaning) && !preamble && !stale) return job;
+    if (!steps.some(isCleaning) && !preamble && !stale) return { ...job, steps };
     const jid = isStr(job.id) ? job.id : "job";
 
     // Walk the Responses steps, turning each into a chunk of items filters
@@ -5802,11 +6040,6 @@ function filtersFromCleaning(next           )            {
     const out              = [];
     let placed = false;
     for (const st of steps) {
-      if (isObj(st) && st.type === "readReply" && isObj(st.out) && "preamble" in st.out) {
-        const { preamble: _p, ...rest } = st.out                           ;
-        out.push({ ...st, out: rest });
-        continue;
-      }
       if (isCleaning(st) || isFilters(st)) {
         if (!placed) { if (textStep) out.push(textStep); if (itemsStep) out.push(itemsStep); placed = true; }
         continue;
@@ -5819,6 +6052,30 @@ function filtersFromCleaning(next           )            {
   return next;
 }
 
+/** [st] without its List `preamble` setting, when it is a Read Reply holding
+    one; [st] itself otherwise. */
+function withoutPreamble(st           )            {
+  if (!isObj(st) || st.type !== "readReply" || !isObj(st.out) || !("preamble" in st.out)) return st;
+  const { preamble: _p, ...out } = st.out                           ;
+  return { ...st, out };
+}
+
+/** Whether a current document still carries a `preamble: false` that an
+    upgrade before #604 left behind. It did nothing, so it is cut as the
+    document is read; a `preamble: true` the lab never wrote at this version is
+    left for validation to refuse. */
+function leftoverPreamble(doc           )          {
+  return Array.isArray(doc.jobs) && doc.jobs.some((job           ) => isObj(job) && Array.isArray(job.steps)
+    && job.steps.some((st           ) => isObj(st) && st.type === "readReply" && isObj(st.out) && st.out.preamble === false));
+}
+
+/** [doc] with each Read Reply's leftover `preamble: false` cut (leftoverPreamble). */
+function withoutLeftoverPreamble(doc           )            {
+  if (!leftoverPreamble(doc)) return doc;
+  return { ...doc, jobs: doc.jobs.map((job           ) => (!isObj(job) || !Array.isArray(job.steps) ? job
+    : { ...job, steps: job.steps.map((st           ) => (isObj(st) && isObj(st.out) && st.out.preamble === false ? withoutPreamble(st) : st)) })) };
+}
+
 /** [doc] with no Metrics eval holding the `case` metric: a dataset's cases
     are metrics now (dataset body version 5), which an eval naming the
     dataset adds for each item, so the case's own metrics carry what that
@@ -5872,6 +6129,24 @@ function localSteps(doc           , ctx            )            {
   return next;
 }
 
+/**
+ * Version 17 to 18 (#596): Saved response pins one response. A Source content
+ * graded as saved -- `live` false, or unset over a Source whose default is
+ * saved, which [ctx] says -- pins the Source's newest successful response
+ * (owner), so a Source holding several grades one where it graded all. Live
+ * data, a content that is not a Source's, or one [ctx] cannot say of, pins
+ * none; a run document carries its files, and needs none.
+ */
+function savedPinned(next           , ctx            )            {
+  next.version = PIPELINE_VERSION;
+  const content = contentOf(next            )             ;
+  if (!isObj(content) || content.type !== "source" || content.live === true || content.pin != null
+      || isObj(next.profiles) || !isRef(content.ref) || !ctx.pinFor) return next;
+  const pin = ctx.pinFor(content.ref.id, typeof content.live === "boolean" ? content.live : undefined);
+  if (pin) content.pin = { name: pin.name, n: pin.n };
+  return next;
+}
+
 /**
  * Version 16 to 17 (#477): a request step has no HTTP API of its own; its
  * job's flow step says it. A step that named another -- Any endpoint over an
@@ -6082,6 +6357,10 @@ function validatePipeline(input         , ctx          = {})           {
     ? (c.sources?.(content.ref?.id)?.files ?? content.files ?? null) : null;
   const nonText = (contentFiles || []).map(f => String((f                             )?.name ?? f))
     .find(n => !/\.(?:txt|md|csv)$/i.test(n));
+  // A document goes whole to the API, so only a type that reads one is
+  // handed it -- never sent as an image or as text it is not.
+  const docType = (contentFiles || []).map(f => documentTypeOf(String((f                             )?.name ?? f)))
+    .find(Boolean) ?? null;
   targets.forEach((_, i) => {
     const conns = doc.jobs.map((_     , k        ) => (targetEntryOf(doc, i, k)?.local ? localConnOf(targetEntryOf(doc, i, k)) : lookup(targetAnswerOf(doc, i, k) )));
     conns.forEach((p         , k        ) => {
@@ -6111,6 +6390,10 @@ function validatePipeline(input         , ctx          = {})           {
         const name = targetAnswerOf(doc, i, k)?.connection.name || type .label;
         bad.push(`${n > 1 ? `${jobLabel(doc, k)}, ` : ""}${targetLabel(doc, i)}: ${name} can't send an ${HTTP_APIS[api]?.label ?? api} request`);
       }
+      if (docType && type && !type.documents && sendsImage(doc.jobs[k])) {
+        const name = targetAnswerOf(doc, i, k)?.connection.name || type.label;
+        bad.push(`${n > 1 ? `${jobLabel(doc, k)}, ` : ""}${targetLabel(doc, i)}: ${name} can't read ${docType.label}s`);
+      }
     });
     // The model is the profile's unless the target's step carries it.
     const bare = conns.findIndex((p                           , k        ) => p && c.profiles && !local(p)
@@ -6317,7 +6600,11 @@ function resolvePipeline(doc          ,
     run.profiles[key] = { ...conn, connection: cid };
   }
   const content = contentOf(run)             ;
-  if (content?.type === "source") content.files = CONTENT_TYPES.source .expand (content, ctx.files);
+  // Saved response grades its one pinned response (#596), whatever else the
+  // Source holds; the server keeps that response's bytes with the run.
+  if (content?.type === "source") {
+    content.files = CONTENT_TYPES.source .expand (content, isObj(content.pin) && content.live !== true ? [content.pin.name] : ctx.files);
+  }
   // Each Library group the evals read, as the version it grades with: the
   // body's fingerprint and, where the lab numbers them, `n` -- a pinned
   // link's pin.
@@ -6584,7 +6871,8 @@ const BUNDLE_DATASET = /^datasets\/([a-z0-9]+(?:-[a-z0-9]+)*)\.json$/;
  * [doc] as a bundle's text files: `pipeline.yaml`, `profiles.yaml` and
  * `datasets/<slug>.json`. [ctx.connections] and [ctx.connectionProfiles]
  * are Setup's -- keys and all, none of which is written -- or
- * [ctx.profiles], Target profiles to split as the store was; [ctx.grader]
+ * [ctx.profiles], Target profiles to split as the store was, and
+ * [ctx.profileDefault] (Default)'s settings; [ctx.grader]
  * the lab's grader, written into each eval that would have asked it, and
  * [ctx.datasets] each graded dataset by id, as `{ name, body }`. Each
  * connection asked is written resolved, under its answer key with the
@@ -6593,12 +6881,13 @@ const BUNDLE_DATASET = /^datasets\/([a-z0-9]+(?:-[a-z0-9]+)*)\.json$/;
  * something else.
  */
 function exportBundle(doc          , ctx                                                                         
-                                                                                                                  
-                                                                                                            = {}) 
+                                                                                                                               
+                                                                                                                                 = {}) 
                                                {
   const split = (ctx.profiles ?? []).map(splitProfile);
   const profiles = withSlugs([...(ctx.connections ?? []), ...split.map(x => x.connection)]                          );
-  const answers = answersOf(profiles, [...(ctx.connectionProfiles ?? []), ...split.flatMap(x => (x.profile ? [x.profile] : []))]);
+  const answers = answersOf(profiles, [...(ctx.connectionProfiles ?? []), ...split.flatMap(x => (x.profile ? [x.profile] : []))],
+                           ctx.profileDefault);
   // What the lab would supply at submit, written down: no lab supplies it later.
   const out = clone(doc)             ;
   out.evals = (Array.isArray(out.evals) ? out.evals : [])
@@ -6774,7 +7063,7 @@ function stagesFor(run             , i        , resolveFilters
     // without a resolver, a Library link reads nothing, a private one its own.
     return { text: targetStepOf(run, i, k) .prompt, kind, withImage: sendsImage(ch),
              verbatim: !!targetEntryOf(run, i, k)?.verbatim, before: beforeFilters(ch, resolveFilters),
-             modifiers: stageModifiers(ch, resolveFilters), settings };
+             after: afterFilters(ch, resolveFilters), modifiers: stageModifiers(ch, resolveFilters), settings };
   });
   // Each named by the connection whose key it is sent with.
   const asked = (key        , held                        ) => {
@@ -6913,7 +7202,9 @@ function scenarioEvals(run             , i        , items
   return evals.map((t, j) => {
     const type = EVAL_TYPES[t.type];
     const whole = isWholeRun(type, t);
-    const base = { id: t.id, label: evalLabel({ evals }, j), whole, skipped: skipRest,
+    // An unnamed eval that links a group goes by the group's name, as the
+    // Evals Summary names it, not "Eval 1" (#519).
+    const base = { id: t.id, label: (t.name || "").trim() || t.group?.name || evalLabel({ evals }, j), whole, skipped: skipRest,
                    verdict: null                          , rules: type?.rules?.(t) ?? [], items: cells.map(() => null)                        ,
                    ran: 0, passed: 0, skippedItems: 0 };
     if (skipRest) {
@@ -7053,6 +7344,13 @@ function validateEvals(ev           , files
       if (scored.length > 1) say("expects its answer discarded, and states something the answer should hold too");
       continue;
     }
+    if (scored.some(m => m.type === "refused" && !m.not)) {
+      // Nor does a call that was refused: there is no answer. Scored in
+      // points, the other metrics are alternatives rather than more to hold
+      // (#560: refused, or the reply says it can't read the file).
+      if (scored.length > 1 && ev.scoring?.mode !== "weighted") say("expects its call refused, and states something the answer should hold too");
+      continue;
+    }
     const wants = scored.filter(m => !m.not && (m.type === "contains-all" || m.type === "contains-any"));
     const forbids = scored.filter(m => m.not && m.type === "contains");
     for (const m of wants) for (const t of values(m)) if (!matchable(t)) say(`expects ${t}, which its own scorer cannot match`);
@@ -7166,7 +7464,7 @@ export {
     loopReplyError, preparedSize,
     termIn, forbiddenIn, readCase, gradedSetFrom, caseItem, caseMetrics, metricLines, upgradeDatasetBody, takeAsExpected, readGroup, readGroupRun, datasetRules, upgradeResults, emptyTally, addToTally, isHostedUrl,
     SEED, REPLY_TOKENS_BEFORE, ANTHROPIC_MAX_TOKENS, asNumber, pinReplyTokens, mappingsFor,
-    CONNECTION_TYPES, HTTP_APIS, httpApiFor, httpRequestOf, httpReplyOf, readFlowReply, WdlError, localAnswer, typeOf, profileType, convertProfile, splitOllama, DECODING_KEYS, apiBase,
+    CONNECTION_TYPES, DOCUMENT_TYPES, documentTypeOf, documentUrl, attachmentOf, HTTP_APIS, httpApiFor, httpRequestOf, httpReplyOf, readFlowReply, WdlError, localAnswer, typeOf, listedConnection, profileType, convertProfile, splitOllama, DECODING_KEYS, apiBase,
     EDGE_448, budgetLabel, IMAGE_FORMATS, encoderQuality,
     tallyPercent, runPipeline, validateEvals, evalsWarnings, evalsJson, readList,
     scoreSingle, singleRules, parseCount, lengthHolds, countHolds, COUNT_OPS, LENGTH_OPS,
@@ -7175,12 +7473,12 @@ export {
     SOURCE_TYPES, DEFAULT_SOURCE_TYPE, sourceTypeOf, WORKFLOW_PLATFORMS, platformOf, DYNAMIC_PROVIDERS, providerOf, WIZARDS, wizardProgress,
     registerKinds, defaultKind, jobDefaults, applyModifiers, keyVar, slugOf, uniqueSlug, isSlug, withSlugs, SLUG_MAX, connectionOf, addressOf, connectionSettings,
     splitProfile, mergeAnswer, answerKey, answerOfKey, answersOf, runAnswer, answerConnectionId, connectionScoped,
-    connectionRequest, connectionBase, connectionSend, connectionReply, connectionReplyProblem, connectionImage, connectionProblems,
+    connectionRequest, connectionBase, connectionSend, connectionReply, connectionReplyProblem, replyFinish, contentJoined, connectionImage, connectionProblems,
 CONNECTION_FIELDS, SETTING_KEYS, OPTION_FIELDS, jobLabel, scenarioLabel, jobWords, versionProblem,
     contentOf, withContent, replyOf,
     targetsOf, targetLetter, targetLabel, targetStepOf, targetEntryOf, jobTargetType, targetAnswerOf, targetAnswerKey, recordedTargetOf, withTarget,
     tokensOf, withTokens, sendsImage, withImage, flowStepOf, requestStepOf, readAsOf, outOf, withOut, slotEntry, SLOTS,
-    filterSetsOf, withFilterSets, stageModifiers, beforeFilters,
+    filterSetsOf, withFilterSets, stageModifiers, beforeFilters, afterFilters, gradedText,
     blankPipeline, upgradePipeline, fatProfileRef, newId, newEval, newLink, validatePipeline, resolvePipeline, pipelineOfRun, stagesFor, profileIds, answerKeys,
     scenarioProfile, lastKind, lastModifier, modifierSummary, itemScores, scenarioEvals, scenarioPasses, groupPass, overallVerdict, passVerdict, isSkipped, failedScore,
     evalLabel, evalsOf, evalsDataset, casesRef, groupOf, groupOfMetrics, pipelineWarnings, itemsFound, previewOf, isWholeRun, EVAL_FIELDS,