evals-lab 0.13.0 → 0.15.0

package/lab/evals-core.mjs

@@ -33,7 +33,7 @@
 // and that is a measurement.
 
 import yaml from "./js-yaml.mjs";
-                                                                                                                                                 
+                                                                                                                                                           
 import { evaluate, asText, templateProblem, WdlError,                 } from "./flows/wdl.mjs";
 import { stepsOf as flowStepsOf, actionsByName, readerFor, readsOf,                          } from "./flows/record.mjs";
 import { flowApi,              } from "./flows/flowApi.mjs";
@@ -41,7 +41,7 @@ import registerMetrics from "./metrics/builtin.mjs";
 import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFilters, preambleFilters, CLEANING_MODIFIERS,
   splitItems, tidy, tryExamples, itemRulesProblems, dropBy, rejectBy, matcher,
   FILTER_TYPES, FILTER_SEPARATORS, FILTER_MODIFIER, applyFilter, applyFilterSet, tryFilter, splitSample, filterProblems, filterFromItemRule,
-                             } from "./kinds/list.mjs";
+  filterV4, V3_TEXT_TYPES,                            } from "./kinds/list.mjs";
 
 // ---- The types -------------------------------------------------------------
 // docs/pipeline-model.md as types, and the shapes of the registries every
@@ -141,16 +141,18 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
                      
  
 
-/** The filter sets a job cleans and gates its reply by (#339, #405): each a
+/** The filter sets a job cleans and gates its reply by (#339, #615): each a
     link to a Library filter set (followed latest or pinned) or a private one
-    held inline, run at the step's `stage` (#432: a set has no stage of its
-    own; the step runs its filters of that stage). A job may hold one `filters` step of
-    each stage -- a `text` one run on the raw reply before it is read into
-    items (Pre-format), an `items` one run on the items after (Post-format). */
+    held inline, one list in run order. `format` is where the job's Response
+    Format reads the reply, when it reads items: before `filterSets[format]`
+    -- the filters above it run on the raw text, those below on each item.
+    Absent, a linked set's own `format` marks it (the first that has one),
+    or else the format reads after them all. A format that reads text reads
+    after them all whatever it says. A job holds one `filters` step. */
                              
                   
-                     
                               
+                  
  
 
 /** A job's steps: its Content stage, then its Responses (§16). */
@@ -199,12 +201,23 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
                                                                                 
                                                                 
                  
+                                                                           
+                                                                         
+                                                                          
+                                                                           
+                           
                                                                             
                                                          
                    
                                 
  
 
+/** One kept response of a dynamic Source's item (#596). */
+                              
+               
+            
+ 
+
                               
                
                
@@ -347,23 +360,20 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
  * at. `filters` are tried in order -- each drop-item narrows the items the
  * next reads, and the first reject-reply that fires gates the answer.
  */
-/** Which side of Response Format a filter runs on (#405): `text` the raw reply before
-    it is read into items (rewrite-text / reject-text), `items` the items after
-    (drop-item, rewrite-item, keep-first, reject-reply). A filter's type decides
-    its stage; a set belongs to none (#432), and runs its share of filters in
-    each stage it is linked in. */
-                                           
-
-/** The stage a filter type's scope belongs to: text filters read the raw
-    reply; item, list and answer filters read the items it is read into. */
-export const stageOfScope = (scope        )              => (scope === "text" ? "text" : "items");
-
-/** A filter stage as the page names it: before the reply is formatted, or after. */
-export const FILTER_STAGE_LABEL                              = { text: "Pre-format", items: "Post-format" };
+/** Whether a job answering as [kind] reads its reply into items, so that
+    where its Response Format sits among its filters matters (#615): the
+    filters above it run on the raw text, those below on each item. A format
+    that reads text reads after every filter. */
+export const formatReadsItems = (kind                           )          =>
+  !!OUTPUT_KINDS[kind ?? ""]?.offers?.includes("items");
 
                                 
                                      
                     
+                                                                        
+                                                                            
+                                                              
+                  
  
 
 /** A Library filter set by reference; a run's copy says the version it used:
@@ -551,9 +561,9 @@ export const FILTER_STAGE_LABEL                              = { text: "Pre-form
                      
                 
                       
+                                                                        
                                                                          
-                                                                           
-                  
+                                                                  
                     
                          
                                                                                    
@@ -586,7 +596,15 @@ export const FILTER_STAGE_LABEL                              = { text: "Pre-form
                                                                         
                                                        
                 
+                                                                          
+                                                                       
+                  
+                      
  
+
+/** A reply the reply-token limit stopped: the limit the request sent, or
+    null where it sent none and the server's own applied. */
+                                                   
 
 /** The transport: a stage's prompt and image in, its reply out. */
                                                                               
@@ -607,9 +625,15 @@ export const FILTER_STAGE_LABEL                              = { text: "Pre-form
                                                                             
                                               
                 
-                                                                            
                                                                          
+                                                                         
+                                             
                                            
+                                                          
+                  
+                                                                           
+                                                                              
+                      
  
 
 /** How much of one text a run keeps -- a reply's body, a grader's prompt or
@@ -630,17 +654,16 @@ export function kept(text        , max = KEPT_CHARS)         {
                   
                       
                                                                            
-                                                                              
-                                                            
+                                                 
                           
                                                                    
                
  
 
-/** Which filter discarded a job's reply, or emptied it, and where (#435). */
+/** Which filter discarded a job's reply, or emptied it, and where (#435). A
+    run kept before #615 also says the stage it ran in, which nothing reads. */
                              
                  
-                     
                                                  
               
                                                                    
@@ -1204,10 +1227,14 @@ const platformOf = (src
                                                                                 
                                                                          
                 
-                                                                            
-                                                                       
-                                                       
+                                                                         
+                                                                             
+                                                                   
                
+                                                                           
+                                                                              
+                           
+                       
                                                                                
                                                 
                                       
@@ -1506,24 +1533,42 @@ function upgradeDatasetBody   (body   )                  {
     `{ version, filters }` body #334 introduces. A later format adds a step to
     `upgradeFilterSetBody`, as `upgradeDatasetBody` grew, with server.py's
     `upgrade_filter_set_body` its twin. */
-const FILTER_SET_VERSION = 3         ;
-
-/** [body] as this version of a filter set (3), from any earlier one. Version 2
-    (#405) named the stage a set shaped; version 3 (#432) names none -- each
-    filter's type decides where it runs -- so a version-1 or -2 body loses its
-    `stage` and reads as this one. Anything else comes back untouched for
-    `filterSetBodyProblems` to refuse. Pure: the same body gives the same
-    answer, and server.py's upgrade_filter_set_body is its twin. */
+const FILTER_SET_VERSION = 4         ;
+
+/** [body] as this version of a filter set (4), from any earlier one. Version 2
+    (#405) named the stage a set shaped, which version 3 (#432) dropped: each
+    filter's type decided where it ran. Version 4 (#615) has seven verbs that
+    run wherever they sit, around a Response Format a set may straddle: each
+    filter becomes the verb that reads a reply as it did (`filterV4`), and a
+    set that held both text and item filters puts its text ones first, in
+    their order, then its `format` marker, then its item ones -- what it ran
+    linked both before and after the format. Anything else comes back
+    untouched for `filterSetBodyProblems` to refuse. Pure: the same body gives
+    the same answer, and server.py's upgrade_filter_set_body is its twin. */
 function upgradeFilterSetBody   (body   )                    {
   if (!isObj(body)) return body;
   if (body.version === FILTER_SET_VERSION) return body;
-  if (body.version === 1 || body.version === 2) {
+  if (body.version === 1 || body.version === 2 || body.version === 3) {
     const { stage: _stage, ...rest } = body;
-    return { ...rest, version: FILTER_SET_VERSION }                            ;
+    if (!Array.isArray(rest.filters)) return { ...rest, version: FILTER_SET_VERSION }                            ;
+    const isText = (f         ) => isObj(f) && V3_TEXT_TYPES.includes(f.type          );
+    const text = rest.filters.filter(isText), items = rest.filters.filter((f         ) => !isText(f));
+    return { ...rest, version: FILTER_SET_VERSION, filters: [...text, ...items].map(filterV4),
+             ...(text.length && items.length ? { format: text.length } : {}) }                            ;
   }
   return body;
 }
 
+/** A version-1 to -3 private filter set (#615) as a version-4 one that runs
+    on the side of the format its link sits: each filter its verb, in its
+    order, and no marker -- a private set held one stage's filters. A current
+    body comes back as it was. */
+function ownBodyV4(own         )          {
+  if (!isObj(own) || ![1, 2, 3].includes(own.version          )) return own;
+  const { stage: _stage, ...rest } = own;
+  return { ...rest, version: FILTER_SET_VERSION, ...(Array.isArray(rest.filters) ? { filters: rest.filters.map(filterV4) } : {}) };
+}
+
 /** A filter set with no filters, at this version: what New filter set starts
     from, and what a lab with none holds. */
 function blankFilterSetBody()                {
@@ -1538,7 +1583,16 @@ function filterSetBodyProblems(body         , at = "the filter set")           {
   if (body.version !== FILTER_SET_VERSION) {
     return [`${at} is version ${JSON.stringify(body.version)}, and the lab reads version ${FILTER_SET_VERSION}`];
   }
-  return filterProblems(body.filters, at);
+  return [...filterProblems(body.filters, at),
+          ...formatProblems(body.format, Array.isArray(body.filters) ? body.filters.length : 0, at)];
+}
+
+/** Why [format] is not a place among [n] entries the Response Format can
+    read at (#615): a whole number from 0 to [n], or none. */
+function formatProblems(format         , n        , at        )           {
+  if (format == null) return [];
+  return Number.isInteger(format) && (format          ) >= 0 && (format          ) <= n
+    ? [] : [`${at}: Response Format has to sit among its filters`];
 }
 
 /** Why [link] is not a filter-set link a run's Responses step may carry, one
@@ -1697,6 +1751,10 @@ function datasetRules(body         )                  {
                                           
                                                                                                        
                                                                        
+                                                                         
+                                                                           
+                                                   
+                                                                                   
                                                                                          
                                                                                                   
                                     
@@ -2106,6 +2164,7 @@ function ollamaType(id        , label        , cloud         )                 {
       const r = j                                                                   ;
       return { raw: r?.message?.content ?? "", finishReason: r?.done_reason ?? null };
     },
+    cutOff: { reason: "length", limit: (body) => asNumber((body.options                                       )?.num_predict) },
     listModels: (base, key) => listModels(base, cloud ? key : null),
     parseModels: idsFrom,
     ...(cloud ? { url: "https://ollama.com" } : { keyless: true }),
@@ -2220,6 +2279,7 @@ CONNECTION_TYPES["openai-compatible"] = {
   documents: true,
   structured: chatSchema,
   parseReply: chatReply,
+  cutOff: { reason: "length", limit: (body) => asNumber(body.max_completion_tokens ?? body.max_tokens) },
   listModels, parseModels: idsFrom,
 };
 
@@ -2271,6 +2331,7 @@ CONNECTION_TYPES.anthropic = {
     const raw = (r?.content || []).filter(b => b.type === "text").map(b => b.text).join("");
     return { raw, finishReason: r?.stop_reason ?? null };
   },
+  cutOff: { reason: "max_tokens", limit: (body) => asNumber(body.max_tokens) },
   // A flow's request, keyed as the Messages API is -- what the HTTP endpoint
   // type's Anthropic template sets. [base] is the /v1 base words hang off;
   // the step's path is the whole path production called, so it goes to the
@@ -2354,6 +2415,7 @@ CONNECTION_TYPES["llama.cpp"] = {
   // llama-server takes OpenAI's response_format, a schema and all.
   structured: chatSchema,
   parseReply: chatReply,
+  cutOff: { reason: "length", limit: (body) => asNumber(body.n_predict) },
   listModels, parseModels: idsFrom,
   // A reply that arrived is still a failure when this profile asked for the
   // Speed looping check (today's speedReplyError, now an opt-in setting).
@@ -2728,6 +2790,18 @@ function connectionReply(conn          , j         )
 }
 
 /** Why a reply [conn] received is a failure though it arrived, or null. */
+/** How a reply finished, as a stage keeps it (#597): the reason [conn]'s
+    API gave, and -- where that reason is its type's reply-token limit -- the
+    limit [body], the request sent, carried. A type that says no reason
+    keeps nothing. */
+function replyFinish(conn          , body         , finishReason               )                                    {
+  if (!finishReason) return {};
+  const cut = typeEntry(conn).cutOff;
+  if (!cut || finishReason !== cut.reason) return { finish: finishReason };
+  const limit = body && typeof body === "object" ? cut.limit(body                           ) : null;
+  return { finish: finishReason, stopped: { limit } };
+}
+
 function connectionReplyProblem(conn          , raw        , finishReason               )                {
   return typeEntry(conn).replyProblem?.(conn, raw, finishReason) ?? null;
 }
@@ -3253,34 +3327,37 @@ async function runPipeline(stages                  , dataUrl               , cal
     total += res.ms;
     transcript.push({ n: i + 1, kind, conn: res.conn ?? null, withImage, sent: res.sent ?? sent,
                       got: res.raw ?? "", ms: res.ms, error: res.error, ...(res.said != null ? { said: res.said } : {}),
-                      ...(res.body != null ? { body: kept(res.body) } : {}) });
+                      ...(res.body != null ? { body: kept(res.body) } : {}),
+                      ...(res.finish ? { finish: res.finish } : {}), ...(res.stopped ? { stopped: res.stopped } : {}) });
     if (res.error) return result({ error: named(res.error) });
     // A stage that failed is the end of the job: nothing after it is asked.
     // Handed nothing, the next stage still answers -- about an empty list --
     // and that answer is graded as though the job had worked.
     const handsOn = i < stages.length - 1;
     const against = res.readAgainst ?? echo;
-    // The Pre-format (text) filters shape the raw reply before the kind
-    // reads it (#405): a Rewrite text reshapes it, a Reject text ends the job.
+    // The filters above the format shape the raw reply before the kind
+    // reads it (#405, #615): they reshape it, or a Reject ends the job.
     const pre = applyFilterSet(st.before || [], { body: res.raw ?? "", items: [], instruction: against }, "text");
-    const by = (filter        , stage             , rejected         )             => ({ filter, stage, job: i + 1, rejected });
-    if (pre.reject != null) return result({ error: named(`discarded: ${pre.reject}`), filteredBy: by(pre.reject, "text", true) });
+    const by = (filter        , rejected         )             => ({ filter, job: i + 1, rejected });
+    if (pre.reject != null) return result({ error: named(`discarded: ${pre.reject}`), filteredBy: by(pre.reject, true) });
     if (pre.rewrote) transcript[transcript.length - 1] .filtered = { got: pre.body, by: pre.rewrote };
     const read = reader.read(pre.body, { echo: against, parsing: opts.parsing, settings: st.settings });
     if (read.error) return result({ error: named(read.error) });
     const mod = applyModifiers(mods[i], kind, read.value, { instruction: against, body: read.body ?? res.raw ?? "" });
     if (mod.reject) {
-      return result({ error: named(`discarded: ${mod.reject}`), ...(mod.by ? { filteredBy: by(mod.by, "items", true) } : {}) });
+      return result({ error: named(`discarded: ${mod.reject}`), ...(mod.by ? { filteredBy: by(mod.by, true) } : {}) });
     }
     // A filter that dropped the last item: what a zero result is explained by.
-    const emptied = mod.by && reader.terms && !reader.terms(mod.value).length ? { filteredBy: by(mod.by, "items", false) } : {};
+    const emptied = mod.by && reader.terms && !reader.terms(mod.value).length ? { filteredBy: by(mod.by, false) } : {};
     const nothing = handsOn ? reader.handsOn?.(mod.value) : null;
     if (nothing) return result({ error: named(nothing), ...emptied });
     const value = mod.value;
     held = { kind, value };
     const gone = [...(read.dropped || []), ...mod.dropped];
     echoed = (read.echoed || 0) + mod.dropped.filter(d => d.by === ECHO).length;
-    done.push({ reply: res.raw ?? "", rendered: reader.render?.(value) || {} });
+    // {reply} and {stageN} hand on the text as its filters left it -- what
+    // was graded -- never the raw reply a filter cleaned (#557).
+    done.push({ reply: transcript[transcript.length - 1] .filtered?.got ?? res.raw ?? "", rendered: reader.render?.(value) || {} });
     if (!handsOn) {
       dropped = gone;
       return result({ echoed, dropped, ...emptied });
@@ -3301,8 +3378,8 @@ function readList(reply        , out                             , instruction =
   const reader = OUTPUT_KINDS[out.kind];
   if (!reader) return { items: [], dropped: [], echoed: 0, error: `"${out.kind}" is not an output kind this lab has` };
   const { kind, modifiers, before, ...settings } = out;
-  // The Pre-format (text) filters shape the raw reply before the kind
-  // reads it (#405); a Reject text ends the job.
+  // The filters above the format shape the raw reply before the kind reads
+  // it (#405, #615); a Reject ends the job.
   const pre = applyFilterSet(before || [], { body: reply, items: [], instruction }, "text");
   if (pre.reject != null) return { items: [], dropped: [], echoed: 0, error: `discarded: ${pre.reject}` };
   const read = reader.read(pre.body, { echo: instruction, settings });
@@ -3380,7 +3457,11 @@ function applyModifiers   (list                                 , kind        ,
 // 16: a target names an AI Connection and, optionally, a Connection Profile,
 // where it named one Target profile; a run's table is keyed by the pair
 // (answerKey) -- the Target profile split in the store as it opens (#445).
-const PIPELINE_VERSION = 17         ;
+// 17: a request reads by its job's HTTP API; 18: a saved-mode Source content
+// pins one response (#596).
+// 19: a job's filters are one list, run in order around its Response Format,
+// and filters are seven verbs (#615).
+const PIPELINE_VERSION = 19         ;
 
 // Plain objects, so an entry is added by assignment and a reader never needs
 // to know which registered it.
@@ -3662,9 +3743,10 @@ function splitProfile(p         )
 }
 
 /** [conn] with [profile]'s settings over it: the one set of settings a
-    request is built from, the shape a Target profile had. A profile sets
-    no connection field. */
-function mergeAnswer(conn                        , profile                           )          {
+    request is built from, the shape a Target profile had. A profile -- or
+    (Default)'s settings, which are one without its id and name -- sets no
+    connection field. */
+function mergeAnswer(conn                        , profile                                      )          {
   const out          = { ...conn };
   const scoped = connectionScoped(typeOf(conn            ));
   for (const [k, v] of Object.entries(profile ?? {})) {
@@ -3681,7 +3763,7 @@ function mergeAnswer(conn                        , profile
  
 
 /** The key a run's table holds [connection] under, with [profile] -- or with
-    none, the lab's defaults. A grader names its connection bare, its id alone,
+    none, (Default)'s settings. A grader names its connection bare, its id alone,
     so it never meets a target's pair (see `answersOf`). */
 const answerKey = (connection        , profile                )         => `${connection}.${profile ?? ""}`;
 
@@ -3716,9 +3798,13 @@ function targetAnswerKey(doc
  * answer key resolved to the settings it sends -- the profile's connection
  * where it names one, else the key's, with the profile's settings over it.
  * A grader's bare id resolves as the Target profile it named did: its
- * connection under the profile split from it. Null where any part is gone.
+ * connection under the profile split from it. One that names no profile, or
+ * a grader with none split beside it, takes [fallback] instead -- (Default)'s
+ * settings, which sit under no named profile, so a profile is still what it
+ * sets (Setup's Used by counts the same, `defaultUses`). Null where any part
+ * is gone.
  */
-function answersOf(connections                         , profiles                              ) {
+function answersOf(connections                         , profiles                              , fallback                  ) {
   const connection = (id        ) => connections.find(c => c?.id === id) ?? null;
   const profile = (id        ) => profiles.find(p => p?.id === id) ?? null;
   const split = (id        )             => {
@@ -3731,7 +3817,7 @@ function answersOf(connections                         , profiles
     const p = pid ? profile(pid) : null;
     if (pid && !p) return null;
     const c = connection(isRef(p?.connection) ? p .connection .id : at.connection);
-    return c ? mergeAnswer(c, p) : null;
+    return c ? mergeAnswer(c, p ?? fallback) : null;
   };
   return { connection, profile, split, resolve };
 }
@@ -3884,15 +3970,16 @@ SOURCE_TYPES.dynamic = {
 };
 
 // HTTP GET (docs/dynamic-sources.md § 1): the safest pull provider. It fetches
-// a URL through the relay server-side -- Snapshot now freezes one Call into the
-// Source, Live produces one fresh at submit -- so the page drives only the
+// a URL through the relay server-side -- Get response keeps a response, Live
+// data fetches it again at submit -- so the page drives only the
 // create dialog and the Call labels; the fetch, redaction and token-refusal are
 // the server's (its SOURCE_PROVIDERS mirror in server.py).
 DYNAMIC_PROVIDERS["http-get"] = {   // provider: the HTTP GET adapter's own id
   id: "http-get",                   // provider: its id, the row's config.provider
   label: "HTTP GET",
   icon: "square-code",
-  noun: "call",
+  noun: "response",
+  produceIcon: "cloud-sync",
   tier: "pull",
   config: ["url"],
   configProblems(config){
@@ -3921,7 +4008,7 @@ DYNAMIC_PROVIDERS["api"] = {      // provider: the API call adapter's own id
   id: "api",                      // provider: its id, the row's config.provider
   label: "API call",
   icon: "arrow-left-right",
-  noun: "call",
+  noun: "response",
   tier: "pull",
   config: ["method", "url", "reqHeaders", "body", "connection"],
   configProblems(config){
@@ -3950,7 +4037,7 @@ DYNAMIC_PROVIDERS["api"] = {      // provider: the API call adapter's own id
 // or an Upload… by hand, formalised as a provider. The lab fetches and executes
 // nothing: a Supplied item is named by the id the caller gives it (its file
 // name), which is what joins it to a case. It has no config and no server-side
-// production, so no Snapshot now and no live; its Calls are the items handed in,
+// production, so no Get response and no live data; its Calls are the items handed in,
 // redacted on the way in like any other (server.py's take_dynamic). A hosted run
 // grades those Calls; a solely-Supplied Source with none is run only via the CLI.
 DYNAMIC_PROVIDERS["supplied"] = {   // provider: the Supplied adapter's own id
@@ -3980,6 +4067,7 @@ DYNAMIC_PROVIDERS["file"] = {   // provider: the File on disk adapter's own id
   label: "File on disk",
   icon: "file",
   noun: "file",
+  produceIcon: "file-input",
   tier: "local",
   config: ["path"],
   configProblems(config){
@@ -4003,6 +4091,7 @@ DYNAMIC_PROVIDERS["command"] = {   // provider: the Command adapter's own id
   label: "Command",
   icon: "terminal",
   noun: "call",
+  produceIcon: "square-terminal",
   tier: "local",
   config: ["command", "args"],
   configProblems(config){
@@ -4085,11 +4174,14 @@ CONTENT_TYPES.source = {
   description: "Every file or record in a Source from the Library.",
   // `live` is a dynamic Source's per-run snapshot-vs-live override
   // (docs/dynamic-sources.md); meaningless for another kind, ignored there.
-  fields: ["type", "ref", "first", "loops", "live"],
+  fields: ["type", "ref", "first", "loops", "live", "pin"],
   runFields: ["files", "revs"],
   validate(c, ctx, bad){
     if (!isRef(c.ref)) return void bad.push("a Source content has to name its Source as { id, name }");
     if (c.live != null && typeof c.live !== "boolean") bad.push("a Source content's live is true or false");
+    if (c.pin != null && !(isObj(c.pin) && isStr(c.pin.name) && c.pin.name && Number.isInteger(c.pin.n) && c.pin.n >= 0)) {
+      bad.push("a Source content's pin names a response as { name, n }");
+    }
     if (c.first != null && !(Number.isInteger(c.first) && c.first > 0)) {
       bad.push("Run first wants a positive number, or empty for all");
     }
@@ -4269,7 +4361,7 @@ function erroredScore                              (s   , error
 }
 
 /** The reply a metric grades and Results shows (#546): the last stage's
-    reply after its Pre-format filters, or the raw reply. */
+    reply after its filters (#557), or the raw reply. */
 function gradedText(res                                                      )         {
   const last = res?.transcript?.at(-1);
   return last?.filtered?.got ?? last?.got ?? res?.raw ?? "";
@@ -5077,23 +5169,20 @@ STEP_TYPES.modifier = {
 };
 
 // A filters step: the filter sets a job cleans and gates its reply by, each a
-// link or a private set (#339, #405), at the step's `stage`. A job may hold
-// one of each stage -- a `text` one Pre-format, an `items` one Post-format -- so
-// the type is `many`; two of a stage is the job's to refuse. A set belongs to
-// no stage (#432): linked here, it runs the filters whose type fits the step's
-// stage and skips the rest. A private set's body is the core's to judge
+// link or a private set (#339), in run order around the job's Response Format
+// (#615, `format`). A private set's body is the core's to judge
 // (filterSetBodyProblems); a link to a Library set is the server's to resolve,
 // so only its shape is checked here.
 STEP_TYPES.filters = {
-  label: "Filter sets", slot: "responses", rank: 2, many: true,
+  label: "Filter sets", slot: "responses", rank: 2,
   description: "Cleans and gates the reply with linked or private filter sets.",
   in: "value", out: "value",
   apply: "runPipeline",
-  fields: ["type", "stage", "filterSets"],
+  fields: ["type", "filterSets", "format"],
   validate(step, _ctx, bad, at){
-    if (step.stage !== "text" && step.stage !== "items") bad.push(`${at}: a filters step shapes the text or the items stage`);
     if (!Array.isArray(step.filterSets)) return void bad.push(`${at}: filterSets has to be a list`);
     for (const link of step.filterSets               ) bad.push(...filterSetLinkProblems(link, at));
+    bad.push(...formatProblems(step.format, step.filterSets.length, at));
   },
 };
 
@@ -5147,6 +5236,22 @@ function withContent                           (doc   , content                )
   return { ...doc, jobs: [withStep(first, "attachContent", content ? { type: "attachContent", content } : null), ...rest] };
 }
 
+/**
+ * Whether target [i]'s words in job [k] have the item's text joined after
+ * them (#597): job 1, a target that asks a prompt, text content -- a Source
+ * of text items, or inline text, as its CONTENT_TYPES entry says -- and no
+ * {text} in the words to place it (textPrompt). A later job's reply goes
+ * only where a token places it, and an image or a document is sent as a part
+ * of its own, so neither is joined. [ctx] reads a Source as validation does.
+ */
+function contentJoined(doc                                                                                  , i        , k        ,
+                       ctx                = {})          {
+  if (k !== 0 || targetEntryOf(doc, i, k)?.asks !== "prompt") return false;
+  if (String(targetStepOf(doc, i, k)?.prompt ?? "").includes("{text}")) return false;
+  const content = contentOf(doc);
+  return !!content && (CONTENT_TYPES[content.type]?.text?.(content, ctx) ?? null) != null;
+}
+
 /** A document's targets, in column order. */
 function targetsOf(doc                                                           )           {
   return Array.isArray(doc?.targets) ? doc .targets             : [];
@@ -5308,70 +5413,112 @@ function withOut(job     , out     )      {
 }
 
 /** [job] reading a reply as `legacyTagsOut` builds it, on the current model
-    (#405): its output a List, a Pre-format text set that reads the reply as
-    the old `preamble` did, and a Post-format items set that cleans and gates
-    it. The filters are `legacyTagsOut`'s, placed as private sets -- what the
-    version-15 upgrade builds for a legacy list job, for a test to build
-    directly. */
+    (#405, #615): its output a List, and one filters list -- a private set
+    above the format that reads the reply as the old `preamble` did, then the
+    format, then a private set that cleans and gates its items. The filters
+    are `legacyTagsOut`'s -- what the upgrade builds for a legacy list job, for
+    a test to build directly. */
 function legacyTagsJob(job     , rules                                     , mode = "LOWER")      {
   const legacy = legacyTagsOut(rules, mode);
   const items = ((legacy.modifiers[0]?.filters            ) || []);
-  let j = withOut(job, { kind: "list", modifiers: [], parse: "csv" });
-  if (legacy.before.length) j = withFilterSets(j, [{ id: `filters-${j.id}-before`, set: null, pin: null, own: { version: FILTER_SET_VERSION, filters: legacy.before } }], "text");
-  if (items.length) j = withFilterSets(j, [{ id: `filters-${j.id}`, set: null, pin: null, own: { version: FILTER_SET_VERSION, filters: items } }], "items");
-  return j;
-}
-
-/** A job's filters step for [stage], when it holds one (#405). */
-const filterStepOf = (job         , stage             )                         =>
-  (job?.steps || []).find((st)                   => isObj(st) && st.type === "filters" && st.stage === stage);
-
-/** The filter-set links a job's Responses stage carries for [stage], in
-    document order (#339, #405): read from that stage's filters step, or none.
-    Defaults to the Post-format (items) stage, the one that was there before. */
-function filterSetsOf(job         , stage              = "items")                  {
-  return (filterStepOf(job, stage)?.filterSets ?? []).slice();
-}
-
-/** [job] with its [stage] filter-set links set to [links]: that stage's
-    filters step added, replaced, or taken away when none are left. The other
-    stage's step is untouched. */
-function withFilterSets(job     , links                          , stage              = "items")      {
-  const steps = job.steps.filter(st => !(st.type === "filters" && st.stage === stage));
-  const next = links.length ? [...steps, { type: "filters"         , stage, filterSets: [...links] }] : steps;
-  return { ...job, steps: ordered(next) };
-}
-
-/** The filters a link resolves to, or none: a private (`own`) link its own
-    body, a Library link the body [resolve] gives (the worker's kept bodies),
-    and a Library link with no body nothing. */
-const linkFilters = (link               , resolve                                                          )           =>
-  (link.own ?? (link.set && resolve ? resolve(link.set) : null))?.filters || [];
-
-/** A job's Post-format (items) Responses as an ordered modifier list for a
-    run (#339): each modifier step as it is, and each items filter-set link as a
-    synthetic modifier that applies the set's filters where the link sits. The
-    Pre-format (text) filters are not modifiers -- `beforeFilters` gives
-    them, and a run applies them to the raw reply before it is read. [resolve]
-    gives a Library link's body. */
+  const j = withOut(job, { kind: "list", modifiers: [], parse: "csv" });
+  const above                  = legacy.before.length
+    ? [{ id: `filters-${j.id}-before`, set: null, pin: null, own: { version: FILTER_SET_VERSION, filters: legacy.before } }] : [];
+  const below                  = items.length
+    ? [{ id: `filters-${j.id}`, set: null, pin: null, own: { version: FILTER_SET_VERSION, filters: items } }] : [];
+  return withFilterSets(j, [...above, ...below], above.length);
+}
+
+/** A job's filters step, when it holds one. */
+const filterStepOf = (job         )                         =>
+  (job?.steps || []).find((st)                   => isObj(st) && st.type === "filters");
+
+/** The filter-set links a job's Responses carries, in run order (#339,
+    #615): read from its filters step, or none. */
+function filterSetsOf(job         )                  {
+  return (filterStepOf(job)?.filterSets ?? []).slice();
+}
+
+/** Where job's own Response Format sits among its links (#615): before
+    `filterSetsOf(job)[n]`, or undefined where it holds no place of its own --
+    a straddling set's marker, or the end, is where it reads. */
+function filterFormatOf(job         )                     {
+  const at = filterStepOf(job)?.format;
+  return Number.isInteger(at) ? at : undefined;
+}
+
+/** [job] with its filter-set links set to [links], and its own Response
+    Format at [format] among them (none: undefined). The filters step is
+    added, replaced, or taken away when no link is left. */
+function withFilterSets(job     , links                          , format         )      {
+  const steps = job.steps.filter(st => st.type !== "filters");
+  const step             = { type: "filters", filterSets: [...links], ...(format != null ? { format } : {}) };
+  return { ...job, steps: ordered(links.length ? [...steps, step] : steps) };
+}
+
+/** The body a link resolves to, or none: a private (`own`) link its own body,
+    a Library link the body [resolve] gives (the worker's kept bodies). */
+const linkBody = (link               , resolve                                                          )                       =>
+  link.own ?? (link.set && resolve ? resolve(link.set) ?? null : null);
+
+/** Where job's Response Format reads among its links (#615), as the index
+    of the link it reads in and the place among that link's filters: its own
+    place (`format`) when it has one, else the first linked set whose body
+    straddles the format (its `format`), else after every link. A format that
+    reads text reads after every link. [resolve] gives a Library link's body. */
+function formatPlace(job         , resolve                                                          )                               {
+  const links = filterSetsOf(job);
+  if (!formatReadsItems(outOf(job)?.kind)) return { link: links.length, at: 0 };
+  const own = filterFormatOf(job);
+  if (own != null) return { link: Math.min(own, links.length), at: 0 };
+  for (let i = 0; i < links.length; i++) {
+    const at = linkBody(links[i] , resolve)?.format;
+    if (Number.isInteger(at)) return { link: i, at: at           };
+  }
+  return { link: links.length, at: 0 };
+}
+
+/** A job's filters on either side of its Response Format (#615): `above`,
+    run on the raw reply before it is read, and `below`, each link's share
+    run on the items after it -- one list a link, so a filter that empties a
+    list is named as a link's always was. */
+function filterSides(job         , resolve                                                          )                                         {
+  const links = filterSetsOf(job);
+  const place = formatPlace(job, resolve);
+  const above           = [], below             = [];
+  links.forEach((link, i) => {
+    const filters = linkBody(link, resolve)?.filters || [];
+    if (i < place.link) above.push(...filters);
+    else if (i > place.link) below.push(filters);
+    else { above.push(...filters.slice(0, place.at)); below.push(filters.slice(place.at)); }
+  });
+  return { above, below };
+}
+
+/** A job's Responses below its Response Format as an ordered modifier list
+    for a run (#339, #615): each modifier step as it is, and each link's share
+    below the format as a synthetic modifier that applies its filters to the
+    items where the filters step sits. The filters above the format are not
+    modifiers -- `beforeFilters` gives them, and a run applies them to the raw
+    reply before it is read. [resolve] gives a Library link's body. */
 function stageModifiers(job         , resolve                                                          )             {
   const out             = [];
   for (const st of job?.steps || []) {
     if (!isObj(st)) continue;
     if (st.type === "modifier") out.push(st.modifier);
-    else if (st.type === "filters" && st.stage === "items") {
-      for (const link of (st.filterSets                   ) || []) out.push({ type: FILTER_MODIFIER, filters: linkFilters(link, resolve) });
+    else if (st.type === "filters") {
+      for (const filters of filterSides(job, resolve).below) out.push({ type: FILTER_MODIFIER, filters });
     }
   }
   return out;
 }
 
-/** A job's Pre-format (text) filters for a run (#405), in document order:
-    the filters of every link on its `text` filters step, which a run applies to
-    the raw reply before the kind reads it into items. [resolve] gives a Library
+/** A job's filters above its Response Format, for a run (#405, #615), in
+    run order: a run applies them to the raw reply before the kind reads it.
+    Every filter of a job whose format reads text. [resolve] gives a Library
     link's body, as `stageModifiers` does. */
 function beforeFilters(job         , resolve                                                          )           {
-  return (filterStepOf(job, "text")?.filterSets || []).flatMap(link => linkFilters(link, resolve));
+  return filterSides(job, resolve).above;
 }
 
 /** A new id: random, so one minted in one browser never collides with one
@@ -5417,11 +5564,6 @@ STEP_TYPES.job = {
     }
     const twice = steps.find((st, x) => !STEP_TYPES[st.type] .many && steps.findIndex(o => o.type === st.type) !== x);
     if (twice) return void bad.push(`${at} has two ${STEP_TYPES[twice.type] .label ?? twice.type} steps`);
-    // A `filters` step is `many`, but only one per stage: a Pre-format and
-    // a Post-format set at most (#405).
-    const twoFilters = steps.find((st, x) => st.type === "filters"
-      && steps.findIndex(o => o.type === "filters" && o.stage === st.stage) !== x);
-    if (twoFilters) return void bad.push(`${at} has two ${twoFilters.stage === "text" ? "Pre-format" : "Post-format"} filter steps`);
     if (steps.some((st, x) => x > 0 && placeOf(st) < placeOf(steps[x - 1]))) {
       return void bad.push(`${at}: steps are its Content, then its Responses, each in order`);
     }
@@ -5533,6 +5675,12 @@ function versionProblem(doc         )                {
                                                                             
                                              
                                                            
+                                                                           
+                                                                           
+                                                                          
+                                                                         
+                           
+                                                                                           
  
 
 /**
@@ -5803,9 +5951,9 @@ function groupsOf(next           )            {
 /** [doc] with each private group's body read as this version of a group
     body (`upgradeDatasetBody`): a version-8 one's `source` goes (#396); and
     each private filter set's as this version of a filter-set body
-    (`upgradeFilterSetBody`): a version-2 one's `stage` goes (#432). A body
-    carries its own version, so this runs on a current pipeline too; one whose
-    bodies are all current comes back as it was. */
+    (`ownBodyV4`): its filters the verbs that read a reply as they did (#615).
+    A body carries its own version, so this runs on a current pipeline too;
+    one whose bodies are all current comes back as it was. */
 function ownBodiesNow(doc           )            {
   const stale = (t         ) => isObj(t) && isObj(t.own) && t.own.version !== DATASET_BODY_VERSION;
   const staleSet = (l         ) => isObj(l) && isObj(l.own) && l.own.version !== FILTER_SET_VERSION;
@@ -5817,14 +5965,15 @@ function ownBodiesNow(doc           )            {
   }
   if (Array.isArray(out.jobs) && out.jobs.some(staleJob)) {
     out = { ...out, jobs: out.jobs.map((j           ) => (!staleJob(j) ? j : { ...j, steps: j.steps.map((st           ) => (!staleStep(st) ? st
-      : { ...st, filterSets: st.filterSets.map((l           ) => (staleSet(l) ? { ...l, own: upgradeFilterSetBody(l.own) } : l)) })) })) };
+      : { ...st, filterSets: st.filterSets.map((l           ) => (staleSet(l) ? { ...l, own: ownBodyV4(l.own) } : l)) })) })) };
   }
   return out;
 }
 
 function upgradePipeline   (doc   , ctx             = {})    {
   // A current document is read as it is, but for a profile reference the Runs
-  // tab saved whole (see profileRefs), which is cut back, and a step on a
+  // tab saved whole (see profileRefs), which is cut back, a Read Reply's
+  // leftover `preamble: false` (withoutLeftoverPreamble), and a step on a
   // profile a target step stands in for (localSteps).
   if (isObj(doc) && doc.version === PIPELINE_VERSION) {
     let out = doc             ;
@@ -5832,12 +5981,20 @@ function upgradePipeline   (doc   , ctx             = {})    {
       out = clone(out)             ;
       profileRefs(out);
     }
-    return localSteps(ownBodiesNow(out), ctx)     ;
+    return localSteps(ownBodiesNow(withoutLeftoverPreamble(out)), ctx)     ;
   }
-  if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16].includes(doc.version          )) return doc;
+  if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18].includes(doc.version          )) return doc;
+  // Every version reads as 18 first, then as 19: a job's filters are one list
+  // around its Response Format (oneFilterList).
+  const nineteen = (d           )    => localSteps(ownBodiesNow(oneFilterList(d)), ctx)     ;
+  if (doc.version === 18) return nineteen(withoutLeftoverPreamble(cutRefs(clone(doc)             )));
+  // Every version reads as 17 first, then as 18: a saved-mode Source content
+  // pins one response (savedPinned).
+  const eighteen = (d           )    => nineteen(savedPinned(d, ctx));
+  if (doc.version === 17) return eighteen(cutRefs(clone(doc)             ));
   // Every version reads as 16 first, then as 17: its requests read by their
   // job's HTTP API (requestsOnJobApi).
-  const seventeen = (d           )    => localSteps(ownBodiesNow(requestsOnJobApi(d)), ctx)     ;
+  const seventeen = (d           )    => eighteen(requestsOnJobApi(d));
   if (doc.version === 16) return seventeen(cutRefs(clone(doc)             ));
   // Every version reads as 15 first, then as 16: its Target profiles split
   // into AI Connections and Connection Profiles (answersFromProfiles).
@@ -5883,11 +6040,13 @@ function filtersFromCleaning(next           )            {
   const isFilters = (st           ) => isObj(st) && st.type === "filters";
   next.jobs = next.jobs.map((job           ) => {
     if (!isObj(job) || !Array.isArray(job.steps)) return job;
-    const steps = job.steps               ;
-    const readReply = steps.find(st => isObj(st) && st.type === "readReply")                         ;
+    const readReply = job.steps.find((st           ) => isObj(st) && st.type === "readReply")                         ;
     const preamble = !!(readReply && isObj(readReply.out) && readReply.out.preamble === true);
+    // Stripped from every Read Reply, `false` included, before the job is
+    // asked whether it has anything to migrate (#604).
+    const steps = (job.steps               ).map(withoutPreamble);
     const stale = steps.some(st => isFilters(st) && (st             ).stage == null);
-    if (!steps.some(isCleaning) && !preamble && !stale) return job;
+    if (!steps.some(isCleaning) && !preamble && !stale) return { ...job, steps };
     const jid = isStr(job.id) ? job.id : "job";
 
     // Walk the Responses steps, turning each into a chunk of items filters
@@ -5902,25 +6061,22 @@ function filtersFromCleaning(next           )            {
       if (isCleaning(st)) buffer = [...buffer, ...cleaningFilters((st             ).modifier            )];
       else if (isFilters(st)) {
         for (const link of ((st             ).filterSets                   ) || []) {
-          if (link.own) buffer = [...buffer, ...(link.own.filters || [])];
+          if (link.own) buffer = [...buffer, ...((ownBodyV4(link.own)                 ).filters || [])];
           else { flush(); links.push(link); }
         }
       }
     }
     flush();
-    const itemsStep                    = links.length ? { type: "filters", stage: "items", filterSets: links } : null;
-    const textStep                    = preamble
+    // Version 18's two filters steps, a stage each, which oneFilterList reads
+    // on the way to 19.
+    const itemsStep                   = links.length ? { type: "filters", stage: "items", filterSets: links } : null;
+    const textStep                   = preamble
       ? { type: "filters", stage: "text", filterSets: [{ id: `filters-${jid}-before`, set: null, pin: null, own: { version: FILTER_SET_VERSION, filters: preambleFilters() } }] }
       : null;
 
     const out              = [];
     let placed = false;
     for (const st of steps) {
-      if (isObj(st) && st.type === "readReply" && isObj(st.out) && "preamble" in st.out) {
-        const { preamble: _p, ...rest } = st.out                           ;
-        out.push({ ...st, out: rest });
-        continue;
-      }
       if (isCleaning(st) || isFilters(st)) {
         if (!placed) { if (textStep) out.push(textStep); if (itemsStep) out.push(itemsStep); placed = true; }
         continue;
@@ -5933,6 +6089,72 @@ function filtersFromCleaning(next           )            {
   return next;
 }
 
+/** Version 18 to 19 (#615): each job's Pre-format and Post-format filters
+    steps become one, in run order -- the Pre-format links, then the
+    Post-format ones -- and where the job's format reads items, its Response
+    Format sits between them (`format`), so every filter runs where it did. A
+    Library set linked on both steps is linked once, between them, and its
+    body (now straddling the format, `upgradeFilterSetBody`) says where the
+    format reads: the job holds no place of its own. A job whose format reads
+    text runs every filter on its text, so it holds no place either. Each
+    private set's body is read as version 4 (`ownBodyV4`). Every version
+    before 19 ends here. */
+function oneFilterList(next           )            {
+  next.version = PIPELINE_VERSION;
+  if (!Array.isArray(next.jobs)) return next;
+  const isFilters = (st         )                  => isObj(st) && st.type === "filters";
+  next.jobs = next.jobs.map((job           ) => {
+    if (!isObj(job) || !Array.isArray(job.steps) || !job.steps.some(isFilters)) return job;
+    const steps = job.steps               ;
+    const linksOf = (stage        ) => steps.filter(st => isFilters(st) && (st.stage ?? "items") === stage)
+      .flatMap(st => (Array.isArray(st.filterSets) ? st.filterSets : [])               )
+      .map(l => (isObj(l) && l.own ? { ...l, own: ownBodyV4(l.own) } : l));
+    const pre = linksOf("text"), post = linksOf("items");
+    const setId = (l           ) => (isObj(l) && isObj(l.set) ? l.set.id           : null);
+    const both = pre.find(l => setId(l) && post.some(o => setId(o) === setId(l) && o.pin === l.pin));
+    let links             , format                    ;
+    if (!formatReadsItems(outOf(job       )?.kind)) { links = [...pre, ...post]; format = undefined; }
+    else if (both) {
+      links = [...pre.filter(l => l !== both), both, ...post.filter(o => !(setId(o) === setId(both) && o.pin === both.pin))];
+      format = undefined;
+    } else { links = [...pre, ...post]; format = post.length ? pre.length : undefined; }
+    const out              = [];
+    let placed = false;
+    for (const st of steps) {
+      if (!isFilters(st)) { out.push(st); continue; }
+      if (placed || !links.length) continue;
+      out.push({ type: "filters", filterSets: links, ...(format != null ? { format } : {}) });
+      placed = true;
+    }
+    return { ...job, steps: out };
+  });
+  return next;
+}
+
+/** [st] without its List `preamble` setting, when it is a Read Reply holding
+    one; [st] itself otherwise. */
+function withoutPreamble(st           )            {
+  if (!isObj(st) || st.type !== "readReply" || !isObj(st.out) || !("preamble" in st.out)) return st;
+  const { preamble: _p, ...out } = st.out                           ;
+  return { ...st, out };
+}
+
+/** Whether a current document still carries a `preamble: false` that an
+    upgrade before #604 left behind. It did nothing, so it is cut as the
+    document is read; a `preamble: true` the lab never wrote at this version is
+    left for validation to refuse. */
+function leftoverPreamble(doc           )          {
+  return Array.isArray(doc.jobs) && doc.jobs.some((job           ) => isObj(job) && Array.isArray(job.steps)
+    && job.steps.some((st           ) => isObj(st) && st.type === "readReply" && isObj(st.out) && st.out.preamble === false));
+}
+
+/** [doc] with each Read Reply's leftover `preamble: false` cut (leftoverPreamble). */
+function withoutLeftoverPreamble(doc           )            {
+  if (!leftoverPreamble(doc)) return doc;
+  return { ...doc, jobs: doc.jobs.map((job           ) => (!isObj(job) || !Array.isArray(job.steps) ? job
+    : { ...job, steps: job.steps.map((st           ) => (isObj(st) && isObj(st.out) && st.out.preamble === false ? withoutPreamble(st) : st)) })) };
+}
+
 /** [doc] with no Metrics eval holding the `case` metric: a dataset's cases
     are metrics now (dataset body version 5), which an eval naming the
     dataset adds for each item, so the case's own metrics carry what that
@@ -5986,6 +6208,24 @@ function localSteps(doc           , ctx            )            {
   return next;
 }
 
+/**
+ * Version 17 to 18 (#596): Saved response pins one response. A Source content
+ * graded as saved -- `live` false, or unset over a Source whose default is
+ * saved, which [ctx] says -- pins the Source's newest successful response
+ * (owner), so a Source holding several grades one where it graded all. Live
+ * data, a content that is not a Source's, or one [ctx] cannot say of, pins
+ * none; a run document carries its files, and needs none.
+ */
+function savedPinned(next           , ctx            )            {
+  next.version = PIPELINE_VERSION;
+  const content = contentOf(next            )             ;
+  if (!isObj(content) || content.type !== "source" || content.live === true || content.pin != null
+      || isObj(next.profiles) || !isRef(content.ref) || !ctx.pinFor) return next;
+  const pin = ctx.pinFor(content.ref.id, typeof content.live === "boolean" ? content.live : undefined);
+  if (pin) content.pin = { name: pin.name, n: pin.n };
+  return next;
+}
+
 /**
  * Version 16 to 17 (#477): a request step has no HTTP API of its own; its
  * job's flow step says it. A step that named another -- Any endpoint over an
@@ -6439,7 +6679,11 @@ function resolvePipeline(doc          ,
     run.profiles[key] = { ...conn, connection: cid };
   }
   const content = contentOf(run)             ;
-  if (content?.type === "source") content.files = CONTENT_TYPES.source .expand (content, ctx.files);
+  // Saved response grades its one pinned response (#596), whatever else the
+  // Source holds; the server keeps that response's bytes with the run.
+  if (content?.type === "source") {
+    content.files = CONTENT_TYPES.source .expand (content, isObj(content.pin) && content.live !== true ? [content.pin.name] : ctx.files);
+  }
   // Each Library group the evals read, as the version it grades with: the
   // body's fingerprint and, where the lab numbers them, `n` -- a pinned
   // link's pin.
@@ -6706,7 +6950,8 @@ const BUNDLE_DATASET = /^datasets\/([a-z0-9]+(?:-[a-z0-9]+)*)\.json$/;
  * [doc] as a bundle's text files: `pipeline.yaml`, `profiles.yaml` and
  * `datasets/<slug>.json`. [ctx.connections] and [ctx.connectionProfiles]
  * are Setup's -- keys and all, none of which is written -- or
- * [ctx.profiles], Target profiles to split as the store was; [ctx.grader]
+ * [ctx.profiles], Target profiles to split as the store was, and
+ * [ctx.profileDefault] (Default)'s settings; [ctx.grader]
  * the lab's grader, written into each eval that would have asked it, and
  * [ctx.datasets] each graded dataset by id, as `{ name, body }`. Each
  * connection asked is written resolved, under its answer key with the
@@ -6715,12 +6960,13 @@ const BUNDLE_DATASET = /^datasets\/([a-z0-9]+(?:-[a-z0-9]+)*)\.json$/;
  * something else.
  */
 function exportBundle(doc          , ctx                                                                         
-                                                                                                                  
-                                                                                                            = {}) 
+                                                                                                                               
+                                                                                                                                 = {}) 
                                                {
   const split = (ctx.profiles ?? []).map(splitProfile);
   const profiles = withSlugs([...(ctx.connections ?? []), ...split.map(x => x.connection)]                          );
-  const answers = answersOf(profiles, [...(ctx.connectionProfiles ?? []), ...split.flatMap(x => (x.profile ? [x.profile] : []))]);
+  const answers = answersOf(profiles, [...(ctx.connectionProfiles ?? []), ...split.flatMap(x => (x.profile ? [x.profile] : []))],
+                           ctx.profileDefault);
   // What the lab would supply at submit, written down: no lab supplies it later.
   const out = clone(doc)             ;
   out.evals = (Array.isArray(out.evals) ? out.evals : [])
@@ -6890,9 +7136,9 @@ function stagesFor(run             , i        , resolveFilters
                                                                                {
   const stages = run.jobs.map((ch, k) => {
     const { kind, modifiers: _mods, ...settings } = outOf(ch);
-    // The modifiers a run applies, with each Post-format filter set a
-    // synthetic modifier where its link sits (#339), and the Pre-format
-    // (text) filters applied to the raw reply before it is read (#405);
+    // The modifiers a run applies, with the filters below the format a
+    // synthetic modifier a link where the filters step sits (#339), and those
+    // above it applied to the raw reply before it is read (#405, #615);
     // without a resolver, a Library link reads nothing, a private one its own.
     return { text: targetStepOf(run, i, k) .prompt, kind, withImage: sendsImage(ch),
              verbatim: !!targetEntryOf(run, i, k)?.verbatim, before: beforeFilters(ch, resolveFilters),
@@ -7178,8 +7424,10 @@ function validateEvals(ev           , files
       continue;
     }
     if (scored.some(m => m.type === "refused" && !m.not)) {
-      // Nor does a call that was refused: there is no answer.
-      if (scored.length > 1) say("expects its call refused, and states something the answer should hold too");
+      // Nor does a call that was refused: there is no answer. Scored in
+      // points, the other metrics are alternatives rather than more to hold
+      // (#560: refused, or the reply says it can't read the file).
+      if (scored.length > 1 && ev.scoring?.mode !== "weighted") say("expects its call refused, and states something the answer should hold too");
       continue;
     }
     const wants = scored.filter(m => !m.not && (m.type === "contains-all" || m.type === "contains-any"));
@@ -7304,12 +7552,12 @@ export {
     SOURCE_TYPES, DEFAULT_SOURCE_TYPE, sourceTypeOf, WORKFLOW_PLATFORMS, platformOf, DYNAMIC_PROVIDERS, providerOf, WIZARDS, wizardProgress,
     registerKinds, defaultKind, jobDefaults, applyModifiers, keyVar, slugOf, uniqueSlug, isSlug, withSlugs, SLUG_MAX, connectionOf, addressOf, connectionSettings,
     splitProfile, mergeAnswer, answerKey, answerOfKey, answersOf, runAnswer, answerConnectionId, connectionScoped,
-    connectionRequest, connectionBase, connectionSend, connectionReply, connectionReplyProblem, connectionImage, connectionProblems,
+    connectionRequest, connectionBase, connectionSend, connectionReply, connectionReplyProblem, replyFinish, contentJoined, connectionImage, connectionProblems,
 CONNECTION_FIELDS, SETTING_KEYS, OPTION_FIELDS, jobLabel, scenarioLabel, jobWords, versionProblem,
     contentOf, withContent, replyOf,
     targetsOf, targetLetter, targetLabel, targetStepOf, targetEntryOf, jobTargetType, targetAnswerOf, targetAnswerKey, recordedTargetOf, withTarget,
     tokensOf, withTokens, sendsImage, withImage, flowStepOf, requestStepOf, readAsOf, outOf, withOut, slotEntry, SLOTS,
-    filterSetsOf, withFilterSets, stageModifiers, beforeFilters, gradedText,
+    filterSetsOf, filterFormatOf, withFilterSets, formatPlace, stageModifiers, beforeFilters, gradedText,
     blankPipeline, upgradePipeline, fatProfileRef, newId, newEval, newLink, validatePipeline, resolvePipeline, pipelineOfRun, stagesFor, profileIds, answerKeys,
     scenarioProfile, lastKind, lastModifier, modifierSummary, itemScores, scenarioEvals, scenarioPasses, groupPass, overallVerdict, passVerdict, isSkipped, failedScore,
     evalLabel, evalsOf, evalsDataset, casesRef, groupOf, groupOfMetrics, pipelineWarnings, itemsFound, previewOf, isWholeRun, EVAL_FIELDS,