evals-lab 0.7.0 → 0.9.0

package/lab/evals-core.mjs

@@ -33,12 +33,14 @@
 // and that is a measurement.
 
 import yaml from "./js-yaml.mjs";
-                                                
+                                                                                                                                                 
 import { evaluate, asText, WdlError,                 } from "./flows/wdl.mjs";
 import { stepsOf as flowStepsOf, actionsByName, readerFor, readsOf,                          } from "./flows/record.mjs";
 import { flowApi,              } from "./flows/flowApi.mjs";
 import registerMetrics from "./metrics/builtin.mjs";
-import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProblems, dropBy, rejectBy, matcher } from "./kinds/list.mjs";
+import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProblems, dropBy, rejectBy, matcher,
+  FILTER_TYPES, FILTER_SEPARATORS, FILTER_MODIFIER, applyFilter, applyFilterSet, tryFilter, splitSample, filterProblems, filterFromItemRule,
+                             } from "./kinds/list.mjs";
 
 // ---- The types -------------------------------------------------------------
 // docs/pipeline-model.md as types, and the shapes of the registries every
@@ -138,9 +140,19 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                      
  
 
+/** The filter sets a job cleans and gates its reply by (#339): each a link to
+    a Library filter set (followed latest or pinned) or a private one held
+    inline. A Responses step beside the modifiers, at the same place, so its
+    sets' filters apply in document order with them -- what the Drop items and
+    Reject the answer modifiers did inline, lifted into linked sets. */
+                             
+                  
+                              
+ 
+
 /** A job's steps: its Content stage, then its Responses (§16). */
                                                                                          
-                                              
+                                                           
 
 /** A target step that prompts a model: its words, resolved under the job's
     token mappings, and a profile of its own where it names one. */
@@ -316,6 +328,45 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
 /** A pipeline's evals, in the order they read a run. */
                            
 
+/**
+ * A filter set (issue #335): a versioned Library body holding the filters
+ * that shape a list reply, mirroring DatasetBody / the eval group. The lab
+ * will keep each as a row and a run grade against the body kept with it, as a
+ * dataset does (#336); for now it is the format a filter set reads and writes
+ * at. `filters` are tried in order -- each drop-item narrows the items the
+ * next reads, and the first reject-reply that fires gates the answer.
+ */
+                                
+                                     
+                    
+ 
+
+/** A Library filter set by reference; a run's copy says the version it used:
+    the body's fingerprint, and its number. Mirrors GroupRef. */
+                                                                  
+
+/** A private filter set: a filter-set body held in the pipeline rather than
+    the Library -- what Runs makes when a link is not to a Library set (#339),
+    as a private eval group (OwnGroup) is. A filter set has no cases, so this
+    is the body itself. */
+                                         
+
+/**
+ * A pipeline's link to a filter set in Responses (issue #335, mirrors
+ * GroupEval / OwnGroup): `set` names a Library set, followed at its newest
+ * version (`pin: null`) or pinned at one (`pin: n`); a private set is `own`,
+ * with `set: null`. From version 14 a job's Responses stage carries these on
+ * a `filters` step (#339); the upgrade from 13 migrates the inline Drop items
+ * / Reject rules into a private one, and the worker cleans and gates a run's
+ * replies against the kept bodies.
+ */
+                                
+             
+                           
+                     
+                     
+ 
+
                            
                                    
                                                                              
@@ -657,6 +708,9 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                                                                               
                                                               
                           
+                                                                              
+                                                          
+                       
                     
                                                                        
                     
@@ -667,9 +721,10 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
 
 /** What a metric needs besides the reply: a model to grade with. */
                             
-                                                                           
-                        
-                                            
+                                                                        
+                                                                        
+                               
+                                                                              
                                                                             
                                                                             
                                                           
@@ -723,6 +778,13 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                     
                                                                                     
                        
+                                                                             
+                                                                           
+                 
+                       
+                                                                             
+                                                  
+                    
  
 
 /** What an earlier eval that failed and does not continue leaves a later one. */
@@ -797,6 +859,9 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                         
                       
                         
+                                                                    
+                                                
+                                                                  
                                                                              
                                                  
 
@@ -860,6 +925,11 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                  
                                                                      
                        
+                                                                              
+                                                                           
+                                                                              
+                                                                              
+                   
                                               
                               
                             
@@ -881,9 +951,9 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                       
                                                                                                    
                                                                        
+                                                                            
                                                                           
-                                                                
-                                                                                            
+                                                                                                                 
  
 
 /** Lookups, and whether it is a run document being validated. */
@@ -1041,7 +1111,7 @@ const platformOf = (src
                                                          
                   
                                                            
-                                                                           
+                                                                                                             
                                                                            
                        
                                                 
@@ -1219,6 +1289,57 @@ function upgradeDatasetBody   (body   )                  {
   return recordedIdsV7(groupOfV6({ source: null, cases: raw.map(caseOfV4).map(caseOfV5) }));
 }
 
+/** The filter-set body's version (issue #335). Version 1 is the first: the
+    `{ version, filters }` body #334 introduces. A later format adds a step to
+    `upgradeFilterSetBody`, as `upgradeDatasetBody` grew, with server.py's
+    `upgrade_filter_set_body` its twin. */
+const FILTER_SET_VERSION = 1         ;
+
+/** [body] as this version of a filter set (1), from any earlier one. There is
+    no earlier version yet, so a version-1 body reads as it is and anything
+    else comes back untouched for `filterSetBodyProblems` to refuse. Pure: the
+    same body gives the same answer, and server.py's upgrade_filter_set_body is
+    its twin. */
+function upgradeFilterSetBody   (body   )                    {
+  if (!isObj(body)) return body;
+  if (body.version === FILTER_SET_VERSION) return body;
+  return body;
+}
+
+/** A filter set with no filters, at this version: what New filter set starts
+    from, and what a lab with none holds. */
+function blankFilterSetBody()                {
+  return { version: FILTER_SET_VERSION, filters: [] };
+}
+
+/** Why [body] is not a filter set the lab reads, one sentence each: an object
+    at this version whose filters are a valid filter list (filterProblems). For
+    the store (#336) and the pipeline link (#339). */
+function filterSetBodyProblems(body         , at = "the filter set")           {
+  if (!isObj(body)) return [`${at} has to be a JSON object`];
+  if (body.version !== FILTER_SET_VERSION) {
+    return [`${at} is version ${JSON.stringify(body.version)}, and the lab reads version ${FILTER_SET_VERSION}`];
+  }
+  return filterProblems(body.filters, at);
+}
+
+/** Why [link] is not a filter-set link a run's Responses step may carry, one
+    sentence each (#339): an id, exactly one of a Library `set` reference or a
+    private `own` body (whose shape is the core's to judge), and a `pin` that
+    is null or a whole number. A Library set is the server's to resolve, so its
+    body is not checked here. */
+function filterSetLinkProblems(link         , at = "a filter set")           {
+  if (!isObj(link)) return [`${at}: a filter-set link has to be a JSON object`];
+  const bad           = [];
+  if (!isStr(link.id) || !link.id.trim()) bad.push(`${at}: a filter-set link has no id`);
+  const hasSet = isObj(link.set) && isStr(link.set.id);
+  const hasOwn = link.own != null;
+  if (hasSet === hasOwn) bad.push(`${at}: a filter-set link names a Library set or holds a private one, not both`);
+  if (hasOwn) bad.push(...filterSetBodyProblems(link.own, `${at}: the private filter set`));
+  if (link.pin != null && !Number.isInteger(link.pin)) bad.push(`${at}: a pin is a whole number or none`);
+  return bad;
+}
+
 /** A version-6 body as a version-8 eval group: scored All, the lab's
     grader, and no metrics of its own for every item or the whole run --
     what a Metrics eval naming the dataset with none of its own graded, which
@@ -1323,6 +1444,10 @@ function datasetRules(body         )                  {
                   
                
                                                
+                                                                            
+                                                                             
+                                                
+                                      
  
 
 /** An HTTP request as a connection type builds it, for the relay to send. */
@@ -1346,6 +1471,10 @@ function datasetRules(body         )                  {
                        
                                          
                                                                                                                   
+                                                                             
+                                                                            
+                                          
+                                                                                                       
                                                                        
                                                                                          
                                                                                                   
@@ -1354,8 +1483,9 @@ function datasetRules(body         )                  {
                                                                            
                                  
                     
-                                                                           
-                                                                 
+                                                                          
+                                                                               
+                                                                         
                
                                                                              
                                                                          
@@ -1708,6 +1838,8 @@ function ollamaType(id        , label        , cloud         )                 {
         body: { model: conn.model, messages: [message], options, stream: false },
       };
     },
+    // Ollama's own: the schema is the format.
+    structured: (body, schema) => ({ ...body, format: schema }),
     parseReply(j) {
       const r = j                                                                   ;
       return { raw: r?.message?.content ?? "", finishReason: r?.done_reason ?? null };
@@ -1754,7 +1886,7 @@ function recordedRaw(record         )         {
 // the flow reads a reply; `local` is the no-Read-as fallback.
 CONNECTION_TYPES.recorded = {
   id: "recorded", label: "Recorded", settings: [], keyless: true, picker: false, replays: true,
-  description: "Replays the reply production recorded for each call. Sends nothing.",
+  description: "Answers each call with the reply it got when it was recorded. Sends nothing.",
   local: (_item, _sent, record) => recordedRaw(record),
   request: sendsNothing, parseReply: sendsNothing, listModels: sendsNothing, parseModels: () => [],
 };
@@ -1769,19 +1901,25 @@ function localAnswer(conn          , text               , sent        , record
   return { raw, ms: 0, conn: (conn.name                      ) ?? null, ...(text == null ? { readAgainst: "" } : {}) };
 }
 
+/** A chat completions body asking for a reply held to [schema]: OpenAI's
+    response_format, which llama-server and most compatible servers take too. */
+const chatSchema = (body                         , schema                         )                          =>
+  ({ ...body, response_format: { type: "json_schema", json_schema: { name: "reply", strict: true, schema } } });
+
 CONNECTION_TYPES["openai-compatible"] = {
   id: "openai-compatible", label: "OpenAI-compatible",
   description: "OpenAI, OpenRouter, vLLM, LM Studio: any chat completions endpoint.",
   settings: [
-    { key: "temperature", label: "Temperature", control: "text", default: "", hint: "" },
+    // A reasoning model refuses a temperature.
+    { key: "temperature", label: "Temperature", control: "text", default: "", hint: "",
+      appliesTo: (conn) => !/^(gpt-5|o\d)/i.test(String(conn.model ?? "")) },
     { key: "nPredict", label: "Reply tokens", control: "text", default: "", hint: "" },
   ],
-  // /v1/chat/completions, covering OpenAI, OpenRouter, vLLM, LM Studio. The
-  // reasoning-model temperature rule and max_completion_tokens stay with this
-  // type, as the hosted providers reject the fields a llama.cpp server takes.
+  // /v1/chat/completions, covering OpenAI, OpenRouter, vLLM, LM Studio.
+  // max_completion_tokens stays with this type, as the hosted providers
+  // reject the fields a llama.cpp server takes.
   request(conn, prompt, dataUrl, base, key) {
     const hosted = isHostedUrl(base);
-    const isReasoning = /^(gpt-5|o\d)/i.test(String(conn.model ?? ""));
     const temp = asNumber(conn.temperature);
     const predict = asNumber(conn.nPredict);
     return {
@@ -1789,7 +1927,7 @@ CONNECTION_TYPES["openai-compatible"] = {
       headers: bearer(key),
       body: {
         model: conn.model,
-        ...(isReasoning || temp == null ? {} : { temperature: temp }),
+        ...(temp == null ? {} : { temperature: temp }),
         ...(predict == null ? {} : hosted ? { max_completion_tokens: predict } : { max_tokens: predict }),
         messages: [{ role: "user", content: [
           { type: "text", text: prompt },
@@ -1798,15 +1936,23 @@ CONNECTION_TYPES["openai-compatible"] = {
       },
     };
   },
+  structured: chatSchema,
   parseReply: chatReply,
   listModels, parseModels: idsFrom,
 };
 
+/** The Claude models that refuse a sampling setting: Opus from 4.7, and
+    every Opus, Sonnet, Fable and Mythos from 5. Older ones (Opus 4.6, Sonnet
+    4.6, Haiku 4.5) take one. */
+const CLAUDE_FIXED_SAMPLING = /claude-(opus-4-[7-9]|(opus|sonnet|fable|mythos)-[5-9])/i;
+
 CONNECTION_TYPES.anthropic = {
   id: "anthropic", label: "Anthropic",
   description: "Claude, through Anthropic's Messages API.",
+  url: "https://api.anthropic.com",
   settings: [
-    { key: "temperature", label: "Temperature", control: "text", default: "", hint: "" },
+    { key: "temperature", label: "Temperature", control: "text", default: "", hint: "",
+      appliesTo: (conn) => !CLAUDE_FIXED_SAMPLING.test(String(conn.model ?? "")) },
     { key: "maxTokens", label: "Reply tokens", control: "text", default: "", hint: "" },
   ],
   // The Messages API: the x-api-key and anthropic-version headers, base64
@@ -1831,6 +1977,9 @@ CONNECTION_TYPES.anthropic = {
       },
     };
   },
+  // The Messages API's structured outputs. A model that cannot hold one
+  // refuses the request, and the grader asks again without it.
+  structured: (body, schema) => ({ ...body, output_config: { ...(body.output_config                      ), format: { type: "json_schema", schema } } }),
   parseReply(j) {
     const r = j                                                                                ;
     const raw = (r?.content || []).filter(b => b.type === "text").map(b => b.text).join("");
@@ -1903,6 +2052,8 @@ CONNECTION_TYPES["llama.cpp"] = {
       },
     };
   },
+  // llama-server takes OpenAI's response_format, a schema and all.
+  structured: chatSchema,
   parseReply: chatReply,
   listModels, parseModels: idsFrom,
   // A reply that arrived is still a failure when this profile asked for the
@@ -2098,12 +2249,14 @@ function extraHeaders(raw         )                         {
   }
   return out;
 }
-/** An address as a whole-request type hangs paths off it: its origin, no /v1. */
+/** An address as a whole-request type hangs paths off it: its origin, no
+    path. A step's path is the whole path production called, so an address
+    entered with one (`…/v1/messages`) would send it twice. */
 function originBase(raw         )         {
-  let s = String(raw || "").trim().replace(/\/+$/, "");
+  let s = String(raw || "").trim();
   if (!s) return "";
   if (!/^https?:\/\//.test(s)) s = "https://" + s;
-  return s;
+  try { return new URL(s).origin; } catch { return s.replace(/\/+$/, ""); }
 }
 const wholeRequestsOnly = ()        => { throw new Error("an HTTP endpoint is sent an HTTP Request step's request, not a prompt"); };
 CONNECTION_TYPES.http = {
@@ -2230,9 +2383,14 @@ function splitOllama(p         )          {
  * connection carries its type's settings under `options`, so they are merged
  * up before the type reads them.
  */
-function connectionRequest(conn          , prompt        , dataUrl               , base        , key                )              {
+function connectionRequest(conn          , prompt        , dataUrl               , base        , key                ,
+                           schema                                 )              {
   const flat           = { ...conn, ...(conn?.options || {}) };
-  return typeEntry(flat).request(flat, prompt, dataUrl, base, key);
+  const type = typeEntry(flat);
+  // A setting the model refuses is not sent, whatever the profile holds.
+  for (const s of type.settings) if (s.appliesTo && !s.appliesTo(flat)) delete flat[s.key];
+  const req = type.request(flat, prompt, dataUrl, base, key);
+  return schema && type.structured ? { ...req, body: type.structured(req.body                           , schema) } : req;
 }
 
 /** Where [conn]'s requests hang off: its type's own reading of its address,
@@ -2885,7 +3043,7 @@ function applyModifiers   (list                                 , kind        ,
 // 12: a Contains metric's Ignore case holds item by item too.
 // 13: an eval is a link to an eval group, or a group of the pipeline's own,
 // and the document has an overall pass rule (pipeline-model §17).
-const PIPELINE_VERSION = 13         ;
+const PIPELINE_VERSION = 14         ;
 
 // Plain objects, so an entry is added by assignment and a reader never needs
 // to know which registered it.
@@ -2934,6 +3092,15 @@ function registerKinds(k       )       {
                                                                          
                                                      
                                                   
+                                                                              
+                                                                             
+                                                                           
+                                                                           
+                                                                       
+                                                           
+                                                                            
+                                                    
+                                       
  
 
 function pluginHost(pluginId        )             {
@@ -2961,6 +3128,14 @@ function pluginHost(pluginId        )             {
       taken(CONNECTION_TYPES, "connection type", [t.id]);
       CONNECTION_TYPES[t.id] = t;
     },
+    registerWorkflowPlatform(t) {
+      taken(WORKFLOW_PLATFORMS, "workflow platform", [t.id]);
+      WORKFLOW_PLATFORMS[t.id] = t;
+    },
+    registerWizard(w) {
+      taken(WIZARDS, "wizard", [w.id]);
+      WIZARDS[w.id] = w;
+    },
   };
 }
 
@@ -3058,6 +3233,13 @@ function withSlugs                                              (list
 /** Whether [s] is a slug as a profile may hold one. */
 const isSlug = (s         )              => isStr(s) && SLUG.test(s) && s.length <= SLUG_MAX;
 
+/** Where [p] connects: its address, or blank, its type's own -- an Anthropic
+    profile left blank reaches Anthropic. Blank for a type with none means the
+    lab's own Ollama, which the runner and the relay fill in. */
+function addressOf(p                                   )         {
+  return String(p.url ?? "").trim() || CONNECTION_TYPES[typeOf(p            )]?.url || "";
+}
+
 /** A stored profile as a run carries it: its request settings, no key. */
 function connectionOf(p         )             {
   const type = typeOf(p);
@@ -3066,7 +3248,7 @@ function connectionOf(p         )             {
   // temperature is a common field, carried at the top like px/format/quality;
   // the options bag holds the type's own settings only.
   for (const k of settings) if (k !== "temperature" && String(p[k] ?? "").trim()) options[k] = p[k];
-  const out             = { name: p.name || "unnamed", url: p.url || "", model: p.model || "", type };
+  const out             = { name: p.name || "unnamed", url: addressOf(p), model: p.model || "", type };
   if (isSlug(p.slug)) out.slug = p.slug;
   for (const k of ["px", "format", "quality"]         ) if (p[k] != null && p[k] !== "") out[k] = p[k]             ;
   if (String(p.temperature ?? "").trim()) out.temperature = p.temperature                   ;
@@ -3187,6 +3369,70 @@ WORKFLOW_PLATFORMS["power-automate"] = {   // platform: the Power Automate adapt
   render: body => JSON.stringify(body, null, 2),
 };
 
+// ---- Setup Wizards (docs/workflow-sources.md § Wizards) ---------------------
+//
+// A wizard is a registry entry whose steps each say where they happen and a
+// done-when predicate on lab state. Steps tick automatically: `done(lab)` reads
+// the cheap state the page already holds (a workflow Source exists, a pipeline
+// with a Recorded target exists, a run of it is in History), never a poll. One
+// wizard runs at a time, held in the browser-only `promptlab.wizard` store; a
+// header pill shows its progress and the Setup › Wizards section draws the same
+// walk-through. The registry lives here so a plugin registers a wizard through
+// the same host as every other kind (PluginHost.registerWizard, phase 7); the
+// page ships the built-in "Test a workflow" and owns its routes and selectors
+// (web/src/app/wizards.ts). The predicates run with the page's privileges and
+// only read lab state, so a plugin wizard is no new trust (docs/packs.md).
+
+/** The slice of lab state a wizard step's `done` reads: what the page already
+    holds, so a tick costs a predicate and not a request. A reader names a field
+    it needs; a wizard that reads more declares it here. */
+                           
+                                               
+                                                         
+                                                                     
+                                           
+                                                                    
+                                                           
+                                                                     
+                                                           
+ 
+
+                             
+             
+                
+                                                                            
+                 
+                                                                   
+                                                                             
+                
+                                                                             
+                                                                       
+                 
+                                                                    
+                               
+ 
+
+                              
+             
+                
+                                                                      
+                
+                               
+ 
+
+const WIZARDS                              = Object.create(null);
+
+/** How far a wizard has got on [lab]: each step's tick, how many are done, and
+    the first step not yet done -- the one the checklist marks current and Go
+    to step heads for. `current` is -1 once every step is done. */
+function wizardProgress(entry             , lab          )   
+                                                                 
+  {
+  const ticks = entry.steps.map((s) => s.done(lab));
+  const done = ticks.filter(Boolean).length;
+  return { ticks, done, total: entry.steps.length, current: ticks.findIndex((t) => !t) };
+}
+
 CONTENT_TYPES.source = {
   label: "Source",
   description: "Every file or record in a Source from the Library.",
@@ -3364,6 +3610,7 @@ function metricInput(res                    , kase             , more          )
   const last = res.transcript?.at(-1);
   const text = last?.got ?? res.raw ?? "";
   return { text, said: last?.said ?? null, terms: res.terms ?? [], replies: [text], plain: !!more.plain,
+           asked: last?.sent ?? null,
            // A case that has been given its own expected reply ("Take B as
            // expected") is compared with that; otherwise the Source's recorded
            // reply for the item.
@@ -3382,7 +3629,7 @@ function runInput(ress                                       , plain         )
     terms.push(...(r.terms || []));
     ms += r.ms ?? 0;
   }
-  return { text: replies.join("\n"), said: null, terms, replies, plain, error: null, ms, recorded: null, kase: null };
+  return { text: replies.join("\n"), said: null, terms, replies, plain, error: null, ms, recorded: null, asked: null, kase: null };
 }
 
 /** One metric's reading of [input]: `of`, `not` and a failure all applied,
@@ -3643,16 +3890,21 @@ EVAL_TYPES.group = {
     want: [m.not ? "not" : "", metricSummary(m), typeof m.weight === "number" && m.weight !== 1 ? `×${m.weight}` : ""]
       .filter(Boolean).join(" ") })),
   profiles: t => (isObj(t.own) && isRef(t.own.grader) ? [t.own.grader] : isRef(t.grader) ? [t.grader] : []),
-  // A run carries the lab's grader where the group names none and may ask
-  // one: a model-graded metric of its own, or a case's. A link's group is
-  // the Library's, so the run's copy of the link carries it.
+  // A run carries the grader each group asks, so its profile goes with the
+  // run: a link's is the Library group's own, or the lab's where it names
+  // none -- the run's copy of the link carries it. A private group takes the
+  // lab's where it names none and may ask one: a model-graded metric of its
+  // own, or a case's.
   resolve(t, ctx){
-    if (!isRef(ctx.grader)) return t;
-    const grader = { id: ctx.grader.id, name: ctx.grader.name };
+    const ref = (g         )             => (isRef(g) ? { id: g.id, name: g.name } : null);
+    const lab = ref(ctx.grader);
     if (isObj(t.own)) {
-      return !isRef(t.own.grader) && (gradedIn(t.own.every) || isRef(t.own.casesFrom)) ? { ...t, own: { ...t.own, grader } } : t;
+      return lab && !isRef(t.own.grader) && (gradedIn(t.own.every) || isRef(t.own.casesFrom))
+        ? { ...t, own: { ...t.own, grader: lab } } : t;
     }
-    return isRef(t.group) && !isRef(t.grader) ? { ...t, grader } : t;
+    if (!isRef(t.group) || isRef(t.grader)) return t;
+    const grader = ref(ctx.groups?.find(g => g.id === (t.group       ).id)?.grader) ?? lab;
+    return grader ? { ...t, grader } : t;
   },
   wholeRun: wholeRunGroup,
   verdict: (t, ress, kind) => {
@@ -3849,6 +4101,9 @@ function metricSummary(m        )         {
     const v = m[o.key];
     if (o.type === "select") return o.choices.find(c => c.value === v)?.label ?? "";
     if (o.type === "checkbox") return !!v === !!fresh[o.key] ? "" : v ? o.label.toLowerCase() : `${o.label.toLowerCase()} off`;
+    // The choices ticked, by their labels, where they are not a new one's.
+    if (o.type === "multi") return !Array.isArray(v) || JSON.stringify(v) === JSON.stringify(fresh[o.key]) ? ""
+      : o.choices.filter(c => v.includes(c.value)).map(c => c.label).join(" or ");
     // Text that runs to lines (a schema) reads as its first words, run together.
     return v == null || isObj(v) ? "" : String(v).replace(/\s+/g, " ").trim().slice(0, 40);
   }).filter(Boolean);
@@ -4014,7 +4269,7 @@ STEP_TYPES.echo = {
 // the flow reads one. No profile, no prompt -- the record is the reply.
 STEP_TYPES.recorded = {
   label: "Recorded", slot: "target", asks: "nothing", local: true, replaces: "recorded",
-  description: "Replays the reply production recorded for each call. Sends nothing.",
+  description: "Answers each call with the reply it got when it was recorded. Sends nothing.",
   firstJobOnly: "a call's recorded reply is job 1's",
   in: "item", out: "text",
   apply: "runPipeline",
@@ -4086,6 +4341,23 @@ STEP_TYPES.modifier = {
   },
 };
 
+// A filters step: the filter sets a job cleans and gates its reply by, each a
+// link or a private set (#339). It sits with the modifiers (rank 2), so its
+// sets apply in document order among them. A private set's body is the core's
+// to judge (filterSetBodyProblems); a link to a Library set is the server's to
+// resolve, so only its shape is checked here.
+STEP_TYPES.filters = {
+  label: "Filter sets", slot: "responses", rank: 2, many: true,
+  description: "Cleans and gates the reply with linked or private filter sets.",
+  in: "value", out: "value",
+  apply: "runPipeline",
+  fields: ["type", "filterSets"],
+  validate(step, _ctx, bad, at){
+    if (!Array.isArray(step.filterSets)) return void bad.push(`${at}: filterSets has to be a list`);
+    for (const link of step.filterSets               ) bad.push(...filterSetLinkProblems(link, at));
+  },
+};
+
 // ---- reading a job's steps, and a target's -----------------------------------
 // Every reader asks these, never a step's position or the document's shape:
 // which steps a job holds is its own, and a version that moves a field
@@ -4281,12 +4553,58 @@ function outOf(job         )      {
   return { ...read, modifiers }       ;
 }
 
-/** [job] reading its reply as [out]: its Format Validation and modifiers. */
+/** [job] reading its reply as [out]: its Format Validation and modifiers.
+    The modifiers replace the job's modifier steps in place, so a modifier step
+    that interleaves with a filters step (#339) keeps its place among them --
+    an edit to one modifier never reorders another step past it. Extra
+    modifiers (one added) come after the last, and a step with no modifier left
+    (one removed) drops out. */
 function withOut(job     , out     )      {
   const { modifiers, ...read } = out;
-  const steps = job.steps.filter(st => st.type !== "readReply" && st.type !== "modifier");
-  return { ...job, steps: ordered([...steps, { type: "readReply", out: read },
-    ...(modifiers || []).map(modifier => ({ type: "modifier", modifier })        )]) };
+  const mods = modifiers || [];
+  let mi = 0;
+  const steps         = [];
+  for (const st of job.steps) {
+    if (st.type === "readReply") continue;        // re-added below, in its place
+    if (st.type === "modifier") { if (mi < mods.length) steps.push({ type: "modifier", modifier: mods[mi++]  }); }
+    else steps.push(st);                           // readAs, filters, content -- kept where they are
+  }
+  for (; mi < mods.length; mi++) steps.push({ type: "modifier", modifier: mods[mi]  });
+  return { ...job, steps: ordered([...steps, { type: "readReply", out: read }]) };
+}
+
+/** The filter-set links a job's Responses stage carries, in document order
+    (#339): read from its one filters step, or none. */
+function filterSetsOf(job         )                  {
+  return (stepOf            (job, "filters")?.filterSets ?? []).slice();
+}
+
+/** [job] with its filter-set links set to [links]: its filters step added,
+    replaced, or taken away when none are left. */
+function withFilterSets(job     , links                          )      {
+  return withStep(job, "filters", links.length ? { type: "filters", filterSets: [...links] } : null);
+}
+
+/** A job's Responses as an ordered modifier list for a run (#339): each
+    modifier step as it is, and each filter-set link as a synthetic modifier
+    that applies the set's filters where the link sits -- so a linked or
+    private set cleans and gates a list reply exactly as the Drop items /
+    Reject modifiers it replaced did, in the same order. [resolve] gives a
+    Library link's body (the worker's kept bodies); a private (`own`) link
+    carries its own, and a Library link with no body resolves to nothing. */
+function stageModifiers(job         , resolve                                                          )             {
+  const out             = [];
+  for (const st of job?.steps || []) {
+    if (!isObj(st)) continue;
+    if (st.type === "modifier") out.push(st.modifier);
+    else if (st.type === "filters") {
+      for (const link of (st.filterSets                   ) || []) {
+        const body = link.own ?? (link.set && resolve ? resolve(link.set) : null);
+        out.push({ type: FILTER_MODIFIER, filters: body?.filters || [] });
+      }
+    }
+  }
+  return out;
 }
 
 /** A new id: random, so one minted in one browser never collides with one
@@ -4528,6 +4846,13 @@ function newLink(group     , name = "")            {
   return { type: "group", id: newId(), name, continueOnFailure: true, group: { id: group.id, name: group.name }, pin: null };
 }
 
+/** A new pipeline link to the Library filter set [set], followed at its newest
+    version. Mirrors newLink; for Runs to use when it links a set in Responses
+    (#339). */
+function newFilterLink(set     )                {
+  return { id: newId(), set: { id: set.id, name: set.name }, pin: null };
+}
+
 function jobsOf(next           )            {
   next.version = PIPELINE_VERSION;
   if (Array.isArray(next.chains)) {
@@ -4674,16 +4999,62 @@ function upgradePipeline   (doc   , ctx             = {})    {
     }
     return localSteps(out, ctx)     ;
   }
-  if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12].includes(doc.version          )) return doc;
+  if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13].includes(doc.version          )) return doc;
+  // Version 13 reads as 14 by migrating a job's inline Drop items / Reject
+  // rules into a private filter set (#339); nothing else about it changes.
+  if (doc.version === 13) {
+    return localSteps(filtersFromRules(clone(doc)             ), ctx)     ;
+  }
   if (doc.version === 11 || doc.version === 12) {
-    return localSteps(groupsOf(withoutCaseMetric(caseAsWritten(cutRefs(clone(doc)             )))), ctx)     ;
+    return localSteps(filtersFromRules(groupsOf(withoutCaseMetric(caseAsWritten(cutRefs(clone(doc)             ))))), ctx)     ;
   }
   // Every version before 10 reads as version 9 first, then as 10, then as 11,
-  // 12 and 13.
+  // 12, 13 and 14.
   // A version-10 document is cut as a current one was (profileRefs); the
   // earlier ones are cut on their way through nineOf.
   const ten = doc.version === 10 ? cutRefs(clone(doc)             ) : targetsFromScenarios(nineOf(clone(doc)             , ctx));
-  return localSteps(groupsOf(withoutCaseMetric(caseAsWritten(evalsKey(ten)))), ctx)     ;
+  return localSteps(filtersFromRules(groupsOf(withoutCaseMetric(caseAsWritten(evalsKey(ten))))), ctx)     ;
+}
+
+/** [doc] with each job's inline Drop items / Reject modifier steps migrated
+    into one private filter set on a `filters` step, version 13 to 14 (#339).
+    Every drop/reject modifier's rules become filters in document order -- a
+    `drop` modifier's rules drop-item filters, a `reject` modifier's
+    reject-reply filters (filterFromItemRule, which carries the matcher
+    unchanged) -- in one `own` set, and the step takes the place of the first
+    of those modifiers, so the reply is cleaned and gated as it was. A job with
+    no such modifier, or only empty ones, is left as it is; this is a no-op on
+    every path that reaches version 14, so it is safe to run on all of them. */
+function filtersFromRules(next           )            {
+  next.version = PIPELINE_VERSION;
+  if (!Array.isArray(next.jobs)) return next;
+  const TYPE                         = { drop: "drop-item", reject: "reject-reply" };
+  const isRule = (st           )          =>
+    isObj(st) && st.type === "modifier" && isObj(st.modifier) && (st.modifier.type          ) in TYPE;
+  next.jobs = next.jobs.map((job           ) => {
+    if (!isObj(job) || !Array.isArray(job.steps) || !job.steps.some(isRule)) return job;
+    const filters           = [];
+    for (const st of job.steps               ) {
+      if (!isRule(st)) continue;
+      const m = (st             ).modifier             ;
+      for (const r of Array.isArray(m.rules) ? m.rules               : []) {
+        filters.push(filterFromItemRule(r, TYPE[m.type          ] ));
+      }
+    }
+    // Empty Drop items / Reject modifiers held nothing, so they simply drop.
+    // The link's id is the job's, so re-upgrading a document mints the same
+    // one (the migration is idempotent) and a run keyed by it reads the same.
+    const link                       = filters.length
+      ? { id: `filters-${isStr(job.id) ? job.id : "job"}`, set: null, pin: null, own: { version: FILTER_SET_VERSION, filters } } : null;
+    let placed = false;
+    const steps              = [];
+    for (const st of job.steps               ) {
+      if (!isRule(st)) { steps.push(st); continue; }
+      if (link && !placed) { steps.push({ type: "filters", filterSets: [link] }); placed = true; }
+    }
+    return { ...job, steps };
+  });
+  return next;
 }
 
 /** [doc] with no Metrics eval holding the `case` metric: a dataset's cases
@@ -5342,7 +5713,8 @@ function exportBundle(doc          , ctx
   // What the lab would supply at submit, written down: no lab supplies it later.
   const out = clone(doc)             ;
   out.evals = (Array.isArray(out.evals) ? out.evals : [])
-    .map((t           ) => EVAL_TYPES[t?.type]?.resolve?.(t, { grader: ctx.grader ?? null }) ?? t);
+    .map((t           ) => EVAL_TYPES[t?.type]?.resolve?.(t, { grader: ctx.grader ?? null,
+      groups: Object.entries(ctx.datasets ?? {}).map(([id, d]) => ({ id, name: d.name, grader: d.body.grader ?? null })) }) ?? t);
   const table                           = {};
   for (const id of profileIds(out)) {
     const p = profiles.find(x => x.id === id);
@@ -5487,13 +5859,16 @@ function readBundle(files             , ctx                                = {})
  * the job it sits in, and the connection it asks -- its own profile, or
  * its scenario's.
  */
-function stagesFor(run             , i        ) 
+function stagesFor(run             , i        , resolveFilters                                                          ) 
                                                                                                         
                                                                                {
   const stages = run.jobs.map((ch, k) => {
-    const { kind, modifiers, ...settings } = outOf(ch);
+    const { kind, modifiers: _mods, ...settings } = outOf(ch);
+    // The modifiers a run applies, with each linked or private filter set a
+    // synthetic modifier where its link sits (#339); without a resolver, a
+    // Library link reads nothing, a private one its own body.
     return { text: targetStepOf(run, i, k) .prompt, kind, withImage: sendsImage(ch),
-             verbatim: !!targetEntryOf(run, i, k)?.verbatim, modifiers: modifiers || [], settings };
+             verbatim: !!targetEntryOf(run, i, k)?.verbatim, modifiers: stageModifiers(ch, resolveFilters), settings };
   });
   const connections = run.jobs.map((_, k) => {
     // A step answered here asks nothing: the profile a run made before it
@@ -5883,16 +6258,20 @@ export {
     scoreSingle, singleRules, parseCount, lengthHolds, countHolds, COUNT_OPS, LENGTH_OPS,
     COUNT_OP_LABEL, LENGTH_OP_LABEL,
     PIPELINE_VERSION, DATASET_BODY_VERSION, STEP_TYPES, CONTENT_TYPES, OUTPUT_KINDS, MODIFIERS, EVAL_TYPES, LEGACY_TESTS, METRICS, readMetrics, metricSummary, itemScoresAsync, recordedReplyOf,
-    SOURCE_TYPES, DEFAULT_SOURCE_TYPE, sourceTypeOf, WORKFLOW_PLATFORMS, platformOf,
-    registerKinds, defaultKind, jobDefaults, applyModifiers, keyVar, slugOf, uniqueSlug, isSlug, withSlugs, SLUG_MAX, connectionOf, connectionSettings,
+    SOURCE_TYPES, DEFAULT_SOURCE_TYPE, sourceTypeOf, WORKFLOW_PLATFORMS, platformOf, WIZARDS, wizardProgress,
+    registerKinds, defaultKind, jobDefaults, applyModifiers, keyVar, slugOf, uniqueSlug, isSlug, withSlugs, SLUG_MAX, connectionOf, addressOf, connectionSettings,
     connectionRequest, connectionBase, connectionSend, connectionReply, connectionReplyProblem, connectionImage, connectionProblems,
 CONNECTION_FIELDS, SETTING_KEYS, OPTION_FIELDS, jobLabel, scenarioLabel, jobWords, versionProblem,
     contentOf, withContent, replyOf,
     targetsOf, targetLetter, targetLabel, targetStepOf, targetEntryOf, jobTargetType, targetProfileOf, recordedTargetOf, withTarget,
     tokensOf, withTokens, sendsImage, withImage, flowStepOf, readAsOf, outOf, withOut, slotEntry, SLOTS,
+    filterSetsOf, withFilterSets, stageModifiers,
     blankPipeline, upgradePipeline, fatProfileRef, newId, newEval, newLink, validatePipeline, resolvePipeline, pipelineOfRun, stagesFor, profileIds,
     scenarioProfile, lastKind, lastModifier, modifierSummary, itemScores, scenarioEvals, scenarioPasses, groupPass, overallVerdict, passVerdict, isSkipped, failedScore,
     evalLabel, evalsOf, evalsDataset, casesRef, groupOf, groupOfMetrics, pipelineWarnings, isWholeRun, EVAL_FIELDS,
     pipelineToYaml, importPipeline, yamlToPipeline, exportBundle, readBundle, bundleProblems, junitXml,
     pluginHost, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProblems, dropBy, rejectBy, matcher,
+    FILTER_SET_VERSION, FILTER_TYPES, FILTER_SEPARATORS, applyFilter, applyFilterSet, tryFilter, splitSample,
+    filterProblems, filterFromItemRule, upgradeFilterSetBody, blankFilterSetBody, filterSetBodyProblems, filterSetLinkProblems,
+    newFilterLink, FILTER_MODIFIER,
 };