evals-lab 0.14.0 → 0.15.0

package/lab/evals-core.mjs

@@ -33,7 +33,7 @@
 // and that is a measurement.
 
 import yaml from "./js-yaml.mjs";
-                                                                                                                                                 
+                                                                                                                                                           
 import { evaluate, asText, templateProblem, WdlError,                 } from "./flows/wdl.mjs";
 import { stepsOf as flowStepsOf, actionsByName, readerFor, readsOf,                          } from "./flows/record.mjs";
 import { flowApi,              } from "./flows/flowApi.mjs";
@@ -41,7 +41,7 @@ import registerMetrics from "./metrics/builtin.mjs";
 import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFilters, preambleFilters, CLEANING_MODIFIERS,
   splitItems, tidy, tryExamples, itemRulesProblems, dropBy, rejectBy, matcher,
   FILTER_TYPES, FILTER_SEPARATORS, FILTER_MODIFIER, applyFilter, applyFilterSet, tryFilter, splitSample, filterProblems, filterFromItemRule,
-                             } from "./kinds/list.mjs";
+  filterV4, V3_TEXT_TYPES,                            } from "./kinds/list.mjs";
 
 // ---- The types -------------------------------------------------------------
 // docs/pipeline-model.md as types, and the shapes of the registries every
@@ -141,16 +141,18 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
                      
  
 
-/** The filter sets a job cleans and gates its reply by (#339, #405): each a
+/** The filter sets a job cleans and gates its reply by (#339, #615): each a
     link to a Library filter set (followed latest or pinned) or a private one
-    held inline, run at the step's `stage` (#432: a set has no stage of its
-    own; the step runs its filters of that stage). A job may hold one `filters` step of
-    each stage -- a `text` one run on the raw reply before it is read into
-    items (Pre-format), an `items` one run on the items after (Post-format). */
+    held inline, one list in run order. `format` is where the job's Response
+    Format reads the reply, when it reads items: before `filterSets[format]`
+    -- the filters above it run on the raw text, those below on each item.
+    Absent, a linked set's own `format` marks it (the first that has one),
+    or else the format reads after them all. A format that reads text reads
+    after them all whatever it says. A job holds one `filters` step. */
                              
                   
-                     
                               
+                  
  
 
 /** A job's steps: its Content stage, then its Responses (§16). */
@@ -358,32 +360,20 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
  * at. `filters` are tried in order -- each drop-item narrows the items the
  * next reads, and the first reject-reply that fires gates the answer.
  */
-/** Which side of Response Format a filter step sits on (#405): `text`
-    before it, on the raw reply (Pre-format), `items` after it (Post-format).
-    It also names what a filter reads: the reply's text (rewrite-text /
-    reject-text) or the items (drop-item, rewrite-item, keep-first,
-    reject-reply). A text filter runs at either stage, an item filter only
-    Post-format after a kind that reads items (#557, `stageReads`); a set
-    belongs to no stage (#432), and runs its share in each it is linked in. */
-                                           
-
-/** The stage a filter type's scope belongs to: text filters read the raw
-    reply; item, list and answer filters read the items it is read into. */
-export const stageOfScope = (scope        )              => (scope === "text" ? "text" : "items");
-
-/** What job's [stage] filters read when it answers as [kind] (#557):
-    Pre-format the text; Post-format the items after a kind that reads
-    items, and the text after any other. A filter runs there when its
-    `stageOfScope` is this. */
-export const stageReads = (stage             , kind                           )              =>
-  stage === "items" && OUTPUT_KINDS[kind ?? ""]?.offers?.includes("items") ? "items" : "text";
-
-/** A filter stage as the page names it: before the reply is formatted, or after. */
-export const FILTER_STAGE_LABEL                              = { text: "Pre-format", items: "Post-format" };
+/** Whether a job answering as [kind] reads its reply into items, so that
+    where its Response Format sits among its filters matters (#615): the
+    filters above it run on the raw text, those below on each item. A format
+    that reads text reads after every filter. */
+export const formatReadsItems = (kind                           )          =>
+  !!OUTPUT_KINDS[kind ?? ""]?.offers?.includes("items");
 
                                 
                                      
                     
+                                                                        
+                                                                            
+                                                              
+                  
  
 
 /** A Library filter set by reference; a run's copy says the version it used:
@@ -571,14 +561,10 @@ export const FILTER_STAGE_LABEL                              = { text: "Pre-form
                      
                 
                       
+                                                                        
                                                                          
-                                                                           
-                  
+                                                                  
                     
-                                                                          
-                                                                          
-                                                                          
-                   
                          
                                                                                    
                                      
@@ -668,17 +654,16 @@ export function kept(text        , max = KEPT_CHARS)         {
                   
                       
                                                                            
-                                                                              
-                                                            
+                                                 
                           
                                                                    
                
  
 
-/** Which filter discarded a job's reply, or emptied it, and where (#435). */
+/** Which filter discarded a job's reply, or emptied it, and where (#435). A
+    run kept before #615 also says the stage it ran in, which nothing reads. */
                              
                  
-                     
                                                  
               
                                                                    
@@ -713,7 +698,6 @@ export function kept(text        , max = KEPT_CHARS)         {
                
                      
                    
-                  
                         
                                     
  
@@ -1247,10 +1231,10 @@ const platformOf = (src
                                                                              
                                                                    
                
-                                                                       
-                                                                        
-                                                                     
-                        
+                                                                           
+                                                                              
+                           
+                       
                                                                                
                                                 
                                       
@@ -1549,24 +1533,42 @@ function upgradeDatasetBody   (body   )                  {
     `{ version, filters }` body #334 introduces. A later format adds a step to
     `upgradeFilterSetBody`, as `upgradeDatasetBody` grew, with server.py's
     `upgrade_filter_set_body` its twin. */
-const FILTER_SET_VERSION = 3         ;
-
-/** [body] as this version of a filter set (3), from any earlier one. Version 2
-    (#405) named the stage a set shaped; version 3 (#432) names none -- each
-    filter's type decides where it runs -- so a version-1 or -2 body loses its
-    `stage` and reads as this one. Anything else comes back untouched for
-    `filterSetBodyProblems` to refuse. Pure: the same body gives the same
-    answer, and server.py's upgrade_filter_set_body is its twin. */
+const FILTER_SET_VERSION = 4         ;
+
+/** [body] as this version of a filter set (4), from any earlier one. Version 2
+    (#405) named the stage a set shaped, which version 3 (#432) dropped: each
+    filter's type decided where it ran. Version 4 (#615) has seven verbs that
+    run wherever they sit, around a Response Format a set may straddle: each
+    filter becomes the verb that reads a reply as it did (`filterV4`), and a
+    set that held both text and item filters puts its text ones first, in
+    their order, then its `format` marker, then its item ones -- what it ran
+    linked both before and after the format. Anything else comes back
+    untouched for `filterSetBodyProblems` to refuse. Pure: the same body gives
+    the same answer, and server.py's upgrade_filter_set_body is its twin. */
 function upgradeFilterSetBody   (body   )                    {
   if (!isObj(body)) return body;
   if (body.version === FILTER_SET_VERSION) return body;
-  if (body.version === 1 || body.version === 2) {
+  if (body.version === 1 || body.version === 2 || body.version === 3) {
     const { stage: _stage, ...rest } = body;
-    return { ...rest, version: FILTER_SET_VERSION }                            ;
+    if (!Array.isArray(rest.filters)) return { ...rest, version: FILTER_SET_VERSION }                            ;
+    const isText = (f         ) => isObj(f) && V3_TEXT_TYPES.includes(f.type          );
+    const text = rest.filters.filter(isText), items = rest.filters.filter((f         ) => !isText(f));
+    return { ...rest, version: FILTER_SET_VERSION, filters: [...text, ...items].map(filterV4),
+             ...(text.length && items.length ? { format: text.length } : {}) }                            ;
   }
   return body;
 }
 
+/** A version-1 to -3 private filter set (#615) as a version-4 one that runs
+    on the side of the format its link sits: each filter its verb, in its
+    order, and no marker -- a private set held one stage's filters. A current
+    body comes back as it was. */
+function ownBodyV4(own         )          {
+  if (!isObj(own) || ![1, 2, 3].includes(own.version          )) return own;
+  const { stage: _stage, ...rest } = own;
+  return { ...rest, version: FILTER_SET_VERSION, ...(Array.isArray(rest.filters) ? { filters: rest.filters.map(filterV4) } : {}) };
+}
+
 /** A filter set with no filters, at this version: what New filter set starts
     from, and what a lab with none holds. */
 function blankFilterSetBody()                {
@@ -1581,7 +1583,16 @@ function filterSetBodyProblems(body         , at = "the filter set")           {
   if (body.version !== FILTER_SET_VERSION) {
     return [`${at} is version ${JSON.stringify(body.version)}, and the lab reads version ${FILTER_SET_VERSION}`];
   }
-  return filterProblems(body.filters, at);
+  return [...filterProblems(body.filters, at),
+          ...formatProblems(body.format, Array.isArray(body.filters) ? body.filters.length : 0, at)];
+}
+
+/** Why [format] is not a place among [n] entries the Response Format can
+    read at (#615): a whole number from 0 to [n], or none. */
+function formatProblems(format         , n        , at        )           {
+  if (format == null) return [];
+  return Number.isInteger(format) && (format          ) >= 0 && (format          ) <= n
+    ? [] : [`${at}: Response Format has to sit among its filters`];
 }
 
 /** Why [link] is not a filter-set link a run's Responses step may carry, one
@@ -3324,30 +3335,20 @@ async function runPipeline(stages                  , dataUrl               , cal
     // and that answer is graded as though the job had worked.
     const handsOn = i < stages.length - 1;
     const against = res.readAgainst ?? echo;
-    // The Pre-format (text) filters shape the raw reply before the kind
-    // reads it (#405): a Rewrite text reshapes it, a Reject text ends the job.
+    // The filters above the format shape the raw reply before the kind
+    // reads it (#405, #615): they reshape it, or a Reject ends the job.
     const pre = applyFilterSet(st.before || [], { body: res.raw ?? "", items: [], instruction: against }, "text");
-    const by = (filter        , stage             , rejected         )             => ({ filter, stage, job: i + 1, rejected });
-    if (pre.reject != null) return result({ error: named(`discarded: ${pre.reject}`), filteredBy: by(pre.reject, "text", true) });
+    const by = (filter        , rejected         )             => ({ filter, job: i + 1, rejected });
+    if (pre.reject != null) return result({ error: named(`discarded: ${pre.reject}`), filteredBy: by(pre.reject, true) });
     if (pre.rewrote) transcript[transcript.length - 1] .filtered = { got: pre.body, by: pre.rewrote };
     const read = reader.read(pre.body, { echo: against, parsing: opts.parsing, settings: st.settings });
     if (read.error) return result({ error: named(read.error) });
     const mod = applyModifiers(mods[i], kind, read.value, { instruction: against, body: read.body ?? res.raw ?? "" });
     if (mod.reject) {
-      return result({ error: named(`discarded: ${mod.reject}`), ...(mod.by ? { filteredBy: by(mod.by, "items", true) } : {}) });
-    }
-    // A kind that reads no items runs its Post-format filters on its text,
-    // once read (#557); what they leave is what is graded.
-    if (st.after?.length && typeof mod.value === "string") {
-      const post = applyFilterSet(st.after, { body: mod.value, items: [], instruction: against }, "text");
-      if (post.reject != null) return result({ error: named(`discarded: ${post.reject}`), filteredBy: by(post.reject, "items", true) });
-      if (post.rewrote) {
-        transcript[transcript.length - 1] .filtered = { got: post.body, by: [...(pre.rewrote || []), ...post.rewrote] };
-        mod.value = post.body;
-      }
+      return result({ error: named(`discarded: ${mod.reject}`), ...(mod.by ? { filteredBy: by(mod.by, true) } : {}) });
     }
     // A filter that dropped the last item: what a zero result is explained by.
-    const emptied = mod.by && reader.terms && !reader.terms(mod.value).length ? { filteredBy: by(mod.by, "items", false) } : {};
+    const emptied = mod.by && reader.terms && !reader.terms(mod.value).length ? { filteredBy: by(mod.by, false) } : {};
     const nothing = handsOn ? reader.handsOn?.(mod.value) : null;
     if (nothing) return result({ error: named(nothing), ...emptied });
     const value = mod.value;
@@ -3377,8 +3378,8 @@ function readList(reply        , out                             , instruction =
   const reader = OUTPUT_KINDS[out.kind];
   if (!reader) return { items: [], dropped: [], echoed: 0, error: `"${out.kind}" is not an output kind this lab has` };
   const { kind, modifiers, before, ...settings } = out;
-  // The Pre-format (text) filters shape the raw reply before the kind
-  // reads it (#405); a Reject text ends the job.
+  // The filters above the format shape the raw reply before the kind reads
+  // it (#405, #615); a Reject ends the job.
   const pre = applyFilterSet(before || [], { body: reply, items: [], instruction }, "text");
   if (pre.reject != null) return { items: [], dropped: [], echoed: 0, error: `discarded: ${pre.reject}` };
   const read = reader.read(pre.body, { echo: instruction, settings });
@@ -3456,7 +3457,11 @@ function applyModifiers   (list                                 , kind        ,
 // 16: a target names an AI Connection and, optionally, a Connection Profile,
 // where it named one Target profile; a run's table is keyed by the pair
 // (answerKey) -- the Target profile split in the store as it opens (#445).
-const PIPELINE_VERSION = 18         ;
+// 17: a request reads by its job's HTTP API; 18: a saved-mode Source content
+// pins one response (#596).
+// 19: a job's filters are one list, run in order around its Response Format,
+// and filters are seven verbs (#615).
+const PIPELINE_VERSION = 19         ;
 
 // Plain objects, so an entry is added by assignment and a reader never needs
 // to know which registered it.
@@ -3965,7 +3970,7 @@ SOURCE_TYPES.dynamic = {
 };
 
 // HTTP GET (docs/dynamic-sources.md § 1): the safest pull provider. It fetches
-// a URL through the relay server-side -- Fetch now keeps a response, Live
+// a URL through the relay server-side -- Get response keeps a response, Live
 // data fetches it again at submit -- so the page drives only the
 // create dialog and the Call labels; the fetch, redaction and token-refusal are
 // the server's (its SOURCE_PROVIDERS mirror in server.py).
@@ -3974,6 +3979,7 @@ DYNAMIC_PROVIDERS["http-get"] = {   // provider: the HTTP GET adapter's own id
   label: "HTTP GET",
   icon: "square-code",
   noun: "response",
+  produceIcon: "cloud-sync",
   tier: "pull",
   config: ["url"],
   configProblems(config){
@@ -4031,7 +4037,7 @@ DYNAMIC_PROVIDERS["api"] = {      // provider: the API call adapter's own id
 // or an Upload… by hand, formalised as a provider. The lab fetches and executes
 // nothing: a Supplied item is named by the id the caller gives it (its file
 // name), which is what joins it to a case. It has no config and no server-side
-// production, so no Fetch now and no live data; its Calls are the items handed in,
+// production, so no Get response and no live data; its Calls are the items handed in,
 // redacted on the way in like any other (server.py's take_dynamic). A hosted run
 // grades those Calls; a solely-Supplied Source with none is run only via the CLI.
 DYNAMIC_PROVIDERS["supplied"] = {   // provider: the Supplied adapter's own id
@@ -4061,7 +4067,7 @@ DYNAMIC_PROVIDERS["file"] = {   // provider: the File on disk adapter's own id
   label: "File on disk",
   icon: "file",
   noun: "file",
-  produceLabel: "Read now",
+  produceIcon: "file-input",
   tier: "local",
   config: ["path"],
   configProblems(config){
@@ -4085,7 +4091,7 @@ DYNAMIC_PROVIDERS["command"] = {   // provider: the Command adapter's own id
   label: "Command",
   icon: "terminal",
   noun: "call",
-  produceLabel: "Run now",
+  produceIcon: "square-terminal",
   tier: "local",
   config: ["command", "args"],
   configProblems(config){
@@ -5163,23 +5169,20 @@ STEP_TYPES.modifier = {
 };
 
 // A filters step: the filter sets a job cleans and gates its reply by, each a
-// link or a private set (#339, #405), at the step's `stage`. A job may hold
-// one of each stage -- a `text` one Pre-format, an `items` one Post-format -- so
-// the type is `many`; two of a stage is the job's to refuse. A set belongs to
-// no stage (#432): linked here, it runs the filters whose type fits the step's
-// stage and skips the rest. A private set's body is the core's to judge
+// link or a private set (#339), in run order around the job's Response Format
+// (#615, `format`). A private set's body is the core's to judge
 // (filterSetBodyProblems); a link to a Library set is the server's to resolve,
 // so only its shape is checked here.
 STEP_TYPES.filters = {
-  label: "Filter sets", slot: "responses", rank: 2, many: true,
+  label: "Filter sets", slot: "responses", rank: 2,
   description: "Cleans and gates the reply with linked or private filter sets.",
   in: "value", out: "value",
   apply: "runPipeline",
-  fields: ["type", "stage", "filterSets"],
+  fields: ["type", "filterSets", "format"],
   validate(step, _ctx, bad, at){
-    if (step.stage !== "text" && step.stage !== "items") bad.push(`${at}: a filters step shapes the text or the items stage`);
     if (!Array.isArray(step.filterSets)) return void bad.push(`${at}: filterSets has to be a list`);
     for (const link of step.filterSets               ) bad.push(...filterSetLinkProblems(link, at));
+    bad.push(...formatProblems(step.format, step.filterSets.length, at));
   },
 };
 
@@ -5410,79 +5413,112 @@ function withOut(job     , out     )      {
 }
 
 /** [job] reading a reply as `legacyTagsOut` builds it, on the current model
-    (#405): its output a List, a Pre-format text set that reads the reply as
-    the old `preamble` did, and a Post-format items set that cleans and gates
-    it. The filters are `legacyTagsOut`'s, placed as private sets -- what the
-    version-15 upgrade builds for a legacy list job, for a test to build
-    directly. */
+    (#405, #615): its output a List, and one filters list -- a private set
+    above the format that reads the reply as the old `preamble` did, then the
+    format, then a private set that cleans and gates its items. The filters
+    are `legacyTagsOut`'s -- what the upgrade builds for a legacy list job, for
+    a test to build directly. */
 function legacyTagsJob(job     , rules                                     , mode = "LOWER")      {
   const legacy = legacyTagsOut(rules, mode);
   const items = ((legacy.modifiers[0]?.filters            ) || []);
-  let j = withOut(job, { kind: "list", modifiers: [], parse: "csv" });
-  if (legacy.before.length) j = withFilterSets(j, [{ id: `filters-${j.id}-before`, set: null, pin: null, own: { version: FILTER_SET_VERSION, filters: legacy.before } }], "text");
-  if (items.length) j = withFilterSets(j, [{ id: `filters-${j.id}`, set: null, pin: null, own: { version: FILTER_SET_VERSION, filters: items } }], "items");
-  return j;
-}
-
-/** A job's filters step for [stage], when it holds one (#405). */
-const filterStepOf = (job         , stage             )                         =>
-  (job?.steps || []).find((st)                   => isObj(st) && st.type === "filters" && st.stage === stage);
-
-/** The filter-set links a job's Responses stage carries for [stage], in
-    document order (#339, #405): read from that stage's filters step, or none.
-    Defaults to the Post-format (items) stage, the one that was there before. */
-function filterSetsOf(job         , stage              = "items")                  {
-  return (filterStepOf(job, stage)?.filterSets ?? []).slice();
-}
-
-/** [job] with its [stage] filter-set links set to [links]: that stage's
-    filters step added, replaced, or taken away when none are left. The other
-    stage's step is untouched. */
-function withFilterSets(job     , links                          , stage              = "items")      {
-  const steps = job.steps.filter(st => !(st.type === "filters" && st.stage === stage));
-  const next = links.length ? [...steps, { type: "filters"         , stage, filterSets: [...links] }] : steps;
-  return { ...job, steps: ordered(next) };
-}
-
-/** The filters a link resolves to, or none: a private (`own`) link its own
-    body, a Library link the body [resolve] gives (the worker's kept bodies),
-    and a Library link with no body nothing. */
-const linkFilters = (link               , resolve                                                          )           =>
-  (link.own ?? (link.set && resolve ? resolve(link.set) : null))?.filters || [];
-
-/** A job's Post-format (items) Responses as an ordered modifier list for a
-    run (#339): each modifier step as it is, and each items filter-set link as a
-    synthetic modifier that applies the set's filters where the link sits. The
-    Pre-format (text) filters are not modifiers -- `beforeFilters` gives
-    them, and a run applies them to the raw reply before it is read. [resolve]
-    gives a Library link's body. */
+  const j = withOut(job, { kind: "list", modifiers: [], parse: "csv" });
+  const above                  = legacy.before.length
+    ? [{ id: `filters-${j.id}-before`, set: null, pin: null, own: { version: FILTER_SET_VERSION, filters: legacy.before } }] : [];
+  const below                  = items.length
+    ? [{ id: `filters-${j.id}`, set: null, pin: null, own: { version: FILTER_SET_VERSION, filters: items } }] : [];
+  return withFilterSets(j, [...above, ...below], above.length);
+}
+
+/** A job's filters step, when it holds one. */
+const filterStepOf = (job         )                         =>
+  (job?.steps || []).find((st)                   => isObj(st) && st.type === "filters");
+
+/** The filter-set links a job's Responses carries, in run order (#339,
+    #615): read from its filters step, or none. */
+function filterSetsOf(job         )                  {
+  return (filterStepOf(job)?.filterSets ?? []).slice();
+}
+
+/** Where job's own Response Format sits among its links (#615): before
+    `filterSetsOf(job)[n]`, or undefined where it holds no place of its own --
+    a straddling set's marker, or the end, is where it reads. */
+function filterFormatOf(job         )                     {
+  const at = filterStepOf(job)?.format;
+  return Number.isInteger(at) ? at : undefined;
+}
+
+/** [job] with its filter-set links set to [links], and its own Response
+    Format at [format] among them (none: undefined). The filters step is
+    added, replaced, or taken away when no link is left. */
+function withFilterSets(job     , links                          , format         )      {
+  const steps = job.steps.filter(st => st.type !== "filters");
+  const step             = { type: "filters", filterSets: [...links], ...(format != null ? { format } : {}) };
+  return { ...job, steps: ordered(links.length ? [...steps, step] : steps) };
+}
+
+/** The body a link resolves to, or none: a private (`own`) link its own body,
+    a Library link the body [resolve] gives (the worker's kept bodies). */
+const linkBody = (link               , resolve                                                          )                       =>
+  link.own ?? (link.set && resolve ? resolve(link.set) ?? null : null);
+
+/** Where job's Response Format reads among its links (#615), as the index
+    of the link it reads in and the place among that link's filters: its own
+    place (`format`) when it has one, else the first linked set whose body
+    straddles the format (its `format`), else after every link. A format that
+    reads text reads after every link. [resolve] gives a Library link's body. */
+function formatPlace(job         , resolve                                                          )                               {
+  const links = filterSetsOf(job);
+  if (!formatReadsItems(outOf(job)?.kind)) return { link: links.length, at: 0 };
+  const own = filterFormatOf(job);
+  if (own != null) return { link: Math.min(own, links.length), at: 0 };
+  for (let i = 0; i < links.length; i++) {
+    const at = linkBody(links[i] , resolve)?.format;
+    if (Number.isInteger(at)) return { link: i, at: at           };
+  }
+  return { link: links.length, at: 0 };
+}
+
+/** A job's filters on either side of its Response Format (#615): `above`,
+    run on the raw reply before it is read, and `below`, each link's share
+    run on the items after it -- one list a link, so a filter that empties a
+    list is named as a link's always was. */
+function filterSides(job         , resolve                                                          )                                         {
+  const links = filterSetsOf(job);
+  const place = formatPlace(job, resolve);
+  const above           = [], below             = [];
+  links.forEach((link, i) => {
+    const filters = linkBody(link, resolve)?.filters || [];
+    if (i < place.link) above.push(...filters);
+    else if (i > place.link) below.push(filters);
+    else { above.push(...filters.slice(0, place.at)); below.push(filters.slice(place.at)); }
+  });
+  return { above, below };
+}
+
+/** A job's Responses below its Response Format as an ordered modifier list
+    for a run (#339, #615): each modifier step as it is, and each link's share
+    below the format as a synthetic modifier that applies its filters to the
+    items where the filters step sits. The filters above the format are not
+    modifiers -- `beforeFilters` gives them, and a run applies them to the raw
+    reply before it is read. [resolve] gives a Library link's body. */
 function stageModifiers(job         , resolve                                                          )             {
   const out             = [];
   for (const st of job?.steps || []) {
     if (!isObj(st)) continue;
     if (st.type === "modifier") out.push(st.modifier);
-    else if (st.type === "filters" && st.stage === "items") {
-      for (const link of (st.filterSets                   ) || []) out.push({ type: FILTER_MODIFIER, filters: linkFilters(link, resolve) });
+    else if (st.type === "filters") {
+      for (const filters of filterSides(job, resolve).below) out.push({ type: FILTER_MODIFIER, filters });
     }
   }
   return out;
 }
 
-/** A job's Post-format filters for a run when its kind reads no items
-    (#557), in document order: every link's on its `items` filters step, which
-    a run applies to the text after the kind reads it. A kind that reads items
-    has none here -- `stageModifiers` runs its Post-format links. */
-function afterFilters(job         , resolve                                                          )           {
-  if (stageReads("items", outOf(job)?.kind) !== "text") return [];
-  return (filterStepOf(job, "items")?.filterSets || []).flatMap(link => linkFilters(link, resolve));
-}
-
-/** A job's Pre-format (text) filters for a run (#405), in document order:
-    the filters of every link on its `text` filters step, which a run applies to
-    the raw reply before the kind reads it into items. [resolve] gives a Library
+/** A job's filters above its Response Format, for a run (#405, #615), in
+    run order: a run applies them to the raw reply before the kind reads it.
+    Every filter of a job whose format reads text. [resolve] gives a Library
     link's body, as `stageModifiers` does. */
 function beforeFilters(job         , resolve                                                          )           {
-  return (filterStepOf(job, "text")?.filterSets || []).flatMap(link => linkFilters(link, resolve));
+  return filterSides(job, resolve).above;
 }
 
 /** A new id: random, so one minted in one browser never collides with one
@@ -5528,11 +5564,6 @@ STEP_TYPES.job = {
     }
     const twice = steps.find((st, x) => !STEP_TYPES[st.type] .many && steps.findIndex(o => o.type === st.type) !== x);
     if (twice) return void bad.push(`${at} has two ${STEP_TYPES[twice.type] .label ?? twice.type} steps`);
-    // A `filters` step is `many`, but only one per stage: a Pre-format and
-    // a Post-format set at most (#405).
-    const twoFilters = steps.find((st, x) => st.type === "filters"
-      && steps.findIndex(o => o.type === "filters" && o.stage === st.stage) !== x);
-    if (twoFilters) return void bad.push(`${at} has two ${twoFilters.stage === "text" ? "Pre-format" : "Post-format"} filter steps`);
     if (steps.some((st, x) => x > 0 && placeOf(st) < placeOf(steps[x - 1]))) {
       return void bad.push(`${at}: steps are its Content, then its Responses, each in order`);
     }
@@ -5920,9 +5951,9 @@ function groupsOf(next           )            {
 /** [doc] with each private group's body read as this version of a group
     body (`upgradeDatasetBody`): a version-8 one's `source` goes (#396); and
     each private filter set's as this version of a filter-set body
-    (`upgradeFilterSetBody`): a version-2 one's `stage` goes (#432). A body
-    carries its own version, so this runs on a current pipeline too; one whose
-    bodies are all current comes back as it was. */
+    (`ownBodyV4`): its filters the verbs that read a reply as they did (#615).
+    A body carries its own version, so this runs on a current pipeline too;
+    one whose bodies are all current comes back as it was. */
 function ownBodiesNow(doc           )            {
   const stale = (t         ) => isObj(t) && isObj(t.own) && t.own.version !== DATASET_BODY_VERSION;
   const staleSet = (l         ) => isObj(l) && isObj(l.own) && l.own.version !== FILTER_SET_VERSION;
@@ -5934,7 +5965,7 @@ function ownBodiesNow(doc           )            {
   }
   if (Array.isArray(out.jobs) && out.jobs.some(staleJob)) {
     out = { ...out, jobs: out.jobs.map((j           ) => (!staleJob(j) ? j : { ...j, steps: j.steps.map((st           ) => (!staleStep(st) ? st
-      : { ...st, filterSets: st.filterSets.map((l           ) => (staleSet(l) ? { ...l, own: upgradeFilterSetBody(l.own) } : l)) })) })) };
+      : { ...st, filterSets: st.filterSets.map((l           ) => (staleSet(l) ? { ...l, own: ownBodyV4(l.own) } : l)) })) })) };
   }
   return out;
 }
@@ -5952,10 +5983,14 @@ function upgradePipeline   (doc   , ctx             = {})    {
     }
     return localSteps(ownBodiesNow(withoutLeftoverPreamble(out)), ctx)     ;
   }
-  if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17].includes(doc.version          )) return doc;
+  if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18].includes(doc.version          )) return doc;
+  // Every version reads as 18 first, then as 19: a job's filters are one list
+  // around its Response Format (oneFilterList).
+  const nineteen = (d           )    => localSteps(ownBodiesNow(oneFilterList(d)), ctx)     ;
+  if (doc.version === 18) return nineteen(withoutLeftoverPreamble(cutRefs(clone(doc)             )));
   // Every version reads as 17 first, then as 18: a saved-mode Source content
   // pins one response (savedPinned).
-  const eighteen = (d           )    => localSteps(ownBodiesNow(savedPinned(d, ctx)), ctx)     ;
+  const eighteen = (d           )    => nineteen(savedPinned(d, ctx));
   if (doc.version === 17) return eighteen(cutRefs(clone(doc)             ));
   // Every version reads as 16 first, then as 17: its requests read by their
   // job's HTTP API (requestsOnJobApi).
@@ -6026,14 +6061,16 @@ function filtersFromCleaning(next           )            {
       if (isCleaning(st)) buffer = [...buffer, ...cleaningFilters((st             ).modifier            )];
       else if (isFilters(st)) {
         for (const link of ((st             ).filterSets                   ) || []) {
-          if (link.own) buffer = [...buffer, ...(link.own.filters || [])];
+          if (link.own) buffer = [...buffer, ...((ownBodyV4(link.own)                 ).filters || [])];
           else { flush(); links.push(link); }
         }
       }
     }
     flush();
-    const itemsStep                    = links.length ? { type: "filters", stage: "items", filterSets: links } : null;
-    const textStep                    = preamble
+    // Version 18's two filters steps, a stage each, which oneFilterList reads
+    // on the way to 19.
+    const itemsStep                   = links.length ? { type: "filters", stage: "items", filterSets: links } : null;
+    const textStep                   = preamble
       ? { type: "filters", stage: "text", filterSets: [{ id: `filters-${jid}-before`, set: null, pin: null, own: { version: FILTER_SET_VERSION, filters: preambleFilters() } }] }
       : null;
 
@@ -6052,6 +6089,48 @@ function filtersFromCleaning(next           )            {
   return next;
 }
 
+/** Version 18 to 19 (#615): each job's Pre-format and Post-format filters
+    steps become one, in run order -- the Pre-format links, then the
+    Post-format ones -- and where the job's format reads items, its Response
+    Format sits between them (`format`), so every filter runs where it did. A
+    Library set linked on both steps is linked once, between them, and its
+    body (now straddling the format, `upgradeFilterSetBody`) says where the
+    format reads: the job holds no place of its own. A job whose format reads
+    text runs every filter on its text, so it holds no place either. Each
+    private set's body is read as version 4 (`ownBodyV4`). Every version
+    before 19 ends here. */
+function oneFilterList(next           )            {
+  next.version = PIPELINE_VERSION;
+  if (!Array.isArray(next.jobs)) return next;
+  const isFilters = (st         )                  => isObj(st) && st.type === "filters";
+  next.jobs = next.jobs.map((job           ) => {
+    if (!isObj(job) || !Array.isArray(job.steps) || !job.steps.some(isFilters)) return job;
+    const steps = job.steps               ;
+    const linksOf = (stage        ) => steps.filter(st => isFilters(st) && (st.stage ?? "items") === stage)
+      .flatMap(st => (Array.isArray(st.filterSets) ? st.filterSets : [])               )
+      .map(l => (isObj(l) && l.own ? { ...l, own: ownBodyV4(l.own) } : l));
+    const pre = linksOf("text"), post = linksOf("items");
+    const setId = (l           ) => (isObj(l) && isObj(l.set) ? l.set.id           : null);
+    const both = pre.find(l => setId(l) && post.some(o => setId(o) === setId(l) && o.pin === l.pin));
+    let links             , format                    ;
+    if (!formatReadsItems(outOf(job       )?.kind)) { links = [...pre, ...post]; format = undefined; }
+    else if (both) {
+      links = [...pre.filter(l => l !== both), both, ...post.filter(o => !(setId(o) === setId(both) && o.pin === both.pin))];
+      format = undefined;
+    } else { links = [...pre, ...post]; format = post.length ? pre.length : undefined; }
+    const out              = [];
+    let placed = false;
+    for (const st of steps) {
+      if (!isFilters(st)) { out.push(st); continue; }
+      if (placed || !links.length) continue;
+      out.push({ type: "filters", filterSets: links, ...(format != null ? { format } : {}) });
+      placed = true;
+    }
+    return { ...job, steps: out };
+  });
+  return next;
+}
+
 /** [st] without its List `preamble` setting, when it is a Read Reply holding
     one; [st] itself otherwise. */
 function withoutPreamble(st           )            {
@@ -7057,13 +7136,13 @@ function stagesFor(run             , i        , resolveFilters
                                                                                {
   const stages = run.jobs.map((ch, k) => {
     const { kind, modifiers: _mods, ...settings } = outOf(ch);
-    // The modifiers a run applies, with each Post-format filter set a
-    // synthetic modifier where its link sits (#339), and the Pre-format
-    // (text) filters applied to the raw reply before it is read (#405);
+    // The modifiers a run applies, with the filters below the format a
+    // synthetic modifier a link where the filters step sits (#339), and those
+    // above it applied to the raw reply before it is read (#405, #615);
     // without a resolver, a Library link reads nothing, a private one its own.
     return { text: targetStepOf(run, i, k) .prompt, kind, withImage: sendsImage(ch),
              verbatim: !!targetEntryOf(run, i, k)?.verbatim, before: beforeFilters(ch, resolveFilters),
-             after: afterFilters(ch, resolveFilters), modifiers: stageModifiers(ch, resolveFilters), settings };
+             modifiers: stageModifiers(ch, resolveFilters), settings };
   });
   // Each named by the connection whose key it is sent with.
   const asked = (key        , held                        ) => {
@@ -7478,7 +7557,7 @@ CONNECTION_FIELDS, SETTING_KEYS, OPTION_FIELDS, jobLabel, scenarioLabel, jobWord
     contentOf, withContent, replyOf,
     targetsOf, targetLetter, targetLabel, targetStepOf, targetEntryOf, jobTargetType, targetAnswerOf, targetAnswerKey, recordedTargetOf, withTarget,
     tokensOf, withTokens, sendsImage, withImage, flowStepOf, requestStepOf, readAsOf, outOf, withOut, slotEntry, SLOTS,
-    filterSetsOf, withFilterSets, stageModifiers, beforeFilters, afterFilters, gradedText,
+    filterSetsOf, filterFormatOf, withFilterSets, formatPlace, stageModifiers, beforeFilters, gradedText,
     blankPipeline, upgradePipeline, fatProfileRef, newId, newEval, newLink, validatePipeline, resolvePipeline, pipelineOfRun, stagesFor, profileIds, answerKeys,
     scenarioProfile, lastKind, lastModifier, modifierSummary, itemScores, scenarioEvals, scenarioPasses, groupPass, overallVerdict, passVerdict, isSkipped, failedScore,
     evalLabel, evalsOf, evalsDataset, casesRef, groupOf, groupOfMetrics, pipelineWarnings, itemsFound, previewOf, isWholeRun, EVAL_FIELDS,