evals-lab 0.7.0 → 0.9.0

package/lab/kinds/list.mjs

@@ -125,21 +125,26 @@ export function dropBy(rules                     , item        )
     `itemCount`, and `share` of the items `items` matches. Every one given must
     hold; a rule that states none never fires. */
 export function rejectBy(rules                     , body        , items                   )                {
-  for (const r of rules) {
-    const m = r.match || {};
-    const conds            = [];
-    if (m.blank === true) conds.push(!body.trim());
-    if (m.itemCount && typeof m.itemCount === "object") conds.push(inRange(m.itemCount         )(items.length));
-    if (m.share && typeof m.share === "object") {
-      const test = matcher(m.items                           );
-      // A share with nothing to measure cannot fire.
-      conds.push(!!test && inRange(m.share         )(items.length ? items.filter(test).length / items.length : 0));
-    }
-    if (conds.length && conds.every(Boolean)) return r.id;
-  }
+  for (const r of rules) if (rejectHolds(r.match || {}, body, items)) return r.id;
   return null;
 }
 
+/** Whether an answer whose text is [body] and items are [items] meets the
+    reject conditions [m]: `blank`, `itemCount` and the `share` of `items` the
+    items match. Every one given must hold; one that states none never fires.
+    The condition half of rejectBy, shared with the reject-reply filter. */
+function rejectHolds(m                         , body        , items                   )          {
+  const conds            = [];
+  if (m.blank === true) conds.push(!body.trim());
+  if (m.itemCount && typeof m.itemCount === "object") conds.push(inRange(m.itemCount         )(items.length));
+  if (m.share && typeof m.share === "object") {
+    const test = matcher(m.items                           );
+    // A share with nothing to measure cannot fire.
+    conds.push(!!test && inRange(m.share         )(items.length ? items.filter(test).length / items.length : 0));
+  }
+  return conds.length > 0 && conds.every(Boolean);
+}
+
 /** Why [rules] are not a rule list [scope] takes, one sentence each. */
 export function itemRulesProblems(rules         , scope                   , at        )           {
   if (!Array.isArray(rules)) return [`${at}: rules has to be a list`];
@@ -184,6 +189,226 @@ export function tryExamples(rule          , scope                   )
   ];
 }
 
+// ---- filters: the generic reply filter (issue #335) ----------------------------
+//
+// A Filter is the reusable unit #334 introduces: it supersedes the inline
+// ItemRule a Drop items or Reject the answer modifier carries today. Where a
+// rule was a bare matcher with examples, a filter has a base TYPE from a
+// registry (not an enum) -- "drop-item" cleans each item of a list reply,
+// "reject-reply" gates the whole answer -- generic conditions reusing the
+// matcher above (ANDed), Options, and Tests that are its own checks. The two
+// base types evaluate here, beside the matcher and rejectHolds they are built
+// from; the Library-stored filter-set body, the pipeline link that carries
+// one, and their version live in evals-core.ts, as DatasetBody and the
+// eval-group link do. Nothing here replaces the Drop items / Reject modifiers:
+// Runs wires a filter set into Responses and migrates the inline rules in a
+// later phase (#339), so both base types and the ItemRule shape stand.
+
+/** How a filter test splits its sample reply into items. "none" keeps the
+    reply whole, for a reject-reply filter that reads the text. */
+export const FILTER_SEPARATORS = ["comma", "lines", "semicolon", "none"]         ;
+                                                                 
+
+/** A sample reply and what the filter should leave of it: the editor splits
+    the sample by `separator`, applies the filter, and shows whether the result
+    reads as `expectedOutput`. A drop-item filter's expected output is the
+    items it keeps; a reject-reply filter's is blank when the reply should be
+    rejected, or left non-blank when it should pass. */
+                             
+                             
+                      
+                         
+ 
+
+/** A filter's Options: keep going when it errors (rather than failing the
+    run), and how many passes it makes over the reply. */
+                                
+                          
+                 
+ 
+
+/** A filter (issue #335): a base TYPE fixed at creation (a FILTER_TYPES id),
+    generic conditions in `match` (the matcher's keys for its type, ANDed),
+    Options and Tests. `id` is its name, as an ItemRule's was. */
+                         
+             
+               
+               
+                                 
+                         
+                      
+ 
+
+/** A reply a filter works over: the text it came as, and the items parsed
+    from it (as the List kind parses a run's reply). */
+                                                              
+
+/** What applying a filter or a set left: the items kept, what was dropped and
+    by which filter, and -- the first reject-reply filter that fired -- the
+    reply's rejection (null when none did). */
+                                                                                             
+
+/** A base filter type (#334's decision: a registry, not an enum). A reader
+    asks the entry by `filter.type`; nothing branches on the id. */
+                                  
+                                                
+                
+                                                                               
+                           
+                                                                      
+                                                            
+                                
+                                                                            
+                                                                 
+                                                           
+ 
+
+/** The lab's base filter types. Generic code asks this registry by
+    `filter.type`; it never names "drop-item" or "reject-reply". */
+export const FILTER_TYPES                                  = {
+  "drop-item": {
+    label: "Drop item", scope: "item", conditions: MATCHER_KEYS,
+    apply(filter, reply) {
+      const test = matcher(filter.match);
+      if (!test) return { items: reply.items, dropped: [], reject: null };
+      const items           = [], dropped            = [];
+      for (const t of reply.items) {
+        if (test(t)) dropped.push({ item: t, by: filter.id });
+        else items.push(t);
+      }
+      return { items, dropped, reject: null };
+    },
+  },
+  "reject-reply": {
+    label: "Reject reply", scope: "answer", conditions: ["blank", "itemCount", "share", "items"],
+    apply(filter, reply) {
+      return { items: reply.items, dropped: [], reject: rejectHolds(filter.match, reply.body, reply.items) ? filter.id : null };
+    },
+  },
+};
+
+/** Apply one filter to [reply], honouring its Options: `repeat` passes over
+    the reply (a reject-reply filter stops at the first that fires), and
+    `ignoreFailures` leaves the reply as it was if the filter errors -- a
+    condition that cannot be read -- rather than failing the run. Pure. */
+export function applyFilter(filter        , reply             )                {
+  const entry = FILTER_TYPES[filter.type];
+  if (!entry) return { items: reply.items, dropped: [], reject: null };
+  const passes = Number.isInteger(filter.options?.repeat) && filter.options.repeat > 0 ? filter.options.repeat : 1;
+  let items = reply.items;
+  const dropped            = [];
+  let reject                = null;
+  for (let pass = 0; pass < passes && reject == null; pass++) {
+    try {
+      const out = entry.apply(filter, { body: reply.body, items });
+      items = out.items;
+      dropped.push(...out.dropped);
+      reject = out.reject;
+    } catch (e) {
+      // Ignore failures: leave the reply as this pass found it; else the run
+      // records the fault.
+      if (!filter.options?.ignoreFailures) throw e;
+      break;
+    }
+  }
+  return { items, dropped, reject };
+}
+
+/** Apply a filter set's filters to [reply], in order: each drop-item filter
+    narrows the items the next reads, and the first reject-reply filter that
+    fires rejects the answer and stops the rest. Pure. */
+export function applyFilterSet(filters                   , reply             )                {
+  let items = reply.items;
+  const dropped            = [];
+  for (const filter of filters) {
+    const out = applyFilter(filter, { body: reply.body, items });
+    items = out.items;
+    dropped.push(...out.dropped);
+    if (out.reject != null) return { items, dropped, reject: out.reject };
+  }
+  return { items, dropped, reject: null };
+}
+
+const SEPARATOR_RE                                         = { comma: /,/, lines: /\n/, semicolon: /;/, none: null };
+
+/** [sample] split into items by [separator]: each trimmed, none blank. "none"
+    keeps the reply whole as one item. */
+export function splitSample(sample        , separator                 )           {
+  const re = SEPARATOR_RE[separator];
+  const parts = re ? String(sample ?? "").split(re) : [String(sample ?? "")];
+  return parts.map(s => s.trim()).filter(Boolean);
+}
+
+/** Each of [filter]'s tests, tried against the filter alone: the sample split
+    by the test's separator, the filter applied, and the result compared with
+    the expected output. A drop-item test passes when the items kept read as
+    the expected items; a reject-reply test passes when the reply is rejected
+    exactly if the expected output is blank. Supersedes tryExamples. */
+export function tryFilter(filter        )                                                   {
+  const scope = FILTER_TYPES[filter.type]?.scope ?? "item";
+  return (filter.tests || []).map(test => {
+    const out = applyFilter(filter, { body: test.sampleInput, items: splitSample(test.sampleInput, test.separator) });
+    if (scope === "answer") {
+      const rejected = out.reject != null;
+      return { test, ok: rejected === !test.expectedOutput.trim(), got: rejected ? "rejected" : out.items.join(", ") };
+    }
+    const want = splitSample(test.expectedOutput, test.separator);
+    const ok = out.items.length === want.length && out.items.every((t, i) => t === want[i]);
+    return { test, ok, got: out.items.join(", ") };
+  });
+}
+
+/** Why [filters] are not a filter list the lab reads, one sentence each: each
+    has a name, no name twice, a base type it has, whole-number repeat of 1 or
+    more, a boolean ignore-failures, and a matcher its type's scope takes
+    (itemRulesProblems). */
+export function filterProblems(filters         , at        )           {
+  if (!Array.isArray(filters)) return [`${at}: filters has to be a list`];
+  const bad           = [], seen = new Set        ();
+  for (const f of filters            ) {
+    const id = f && typeof f.id === "string" && f.id.trim() ? f.id : null;
+    if (!id) { bad.push(`${at}: a filter has no name`); continue; }
+    if (seen.has(id)) bad.push(`${at}: ${id} is named twice`);
+    seen.add(id);
+    const entry = typeof f.type === "string" ? FILTER_TYPES[f.type] : undefined;
+    if (!entry) { bad.push(`${at}: ${id} has no base type`); continue; }
+    const o = f.options;
+    if (!o || typeof o !== "object" || Array.isArray(o)) bad.push(`${at}: ${id} has no options`);
+    else {
+      if (!Number.isInteger(o.repeat) || o.repeat < 1) bad.push(`${at}: ${id}'s repeat has to be a whole number of 1 or more`);
+      if (typeof o.ignoreFailures !== "boolean") bad.push(`${at}: ${id}'s ignore failures has to be true or false`);
+    }
+    bad.push(...itemRulesProblems([{ id, match: f.match }], entry.scope, at));
+  }
+  return bad;
+}
+
+/** The filter an inline ItemRule becomes (issue #335): its name, why and
+    matcher kept as they are, its base [type] the one the modifier carrying it
+    fixed ("drop-item" for Drop items, "reject-reply" for Reject the answer),
+    its Options the defaults, and its examples its tests -- a caught example
+    becomes a sample the filter should empty (drop-item) or reject
+    (reject-reply), a kept example one it should leave. Pure: the match is
+    carried unchanged, so the filter cleans a reply exactly as the rule did.
+    The only reader of the old shape, for Runs to migrate its inline rules
+    (#339). */
+export function filterFromItemRule(rule          , type        )         {
+  const scope = FILTER_TYPES[type]?.scope ?? "item";
+  const ex = rule.examples || {};
+  const caught = scope === "item" ? ex.drops || [] : ex.rejects || [];
+  const tests               = [
+    ...caught.map((s)             => ({ separator: "comma", sampleInput: s, expectedOutput: "" })),
+    ...(ex.keeps || []).map((s)             => ({ separator: "comma", sampleInput: s, expectedOutput: s })),
+  ];
+  return {
+    id: rule.id, type,
+    ...(rule.why != null ? { why: rule.why } : {}),
+    match: rule.match ? { ...rule.match } : {},
+    options: { ignoreFailures: false, repeat: 1 },
+    tests,
+  };
+}
+
 // ---- the modifiers -------------------------------------------------------------
 
 /** What an item's numbering looks like: "[1] suv", "(2) car", "3. dirt",
@@ -225,9 +450,28 @@ export function applyCase(item        , mode        )         {
   }
 }
 
+/** The id of the synthetic modifier a linked or private filter set applies
+    under at run time (#339): it is never in a document or the Add menu -- a
+    job carries a `filters` step, which `stageModifiers` turns into one of
+    these where the step sits, so a set cleans and gates a list reply exactly
+    as the Drop items / Reject modifiers it replaced did, in the same order. */
+export const FILTER_MODIFIER = "filterSet";
+
 const LIST_MODIFIERS                                       = {
+  // A filter set's filters, applied where its link sits: the run's only reader
+  // of a resolved set, hidden from the Add menu (no options/defaults) because
+  // a set is linked in Responses, not added as a modifier.
+  [FILTER_MODIFIER]: {
+    accepts: ["list"], hidden: true,
+    apply(items, m, ctx) {
+      const out = applyFilterSet((m.filters            ) || [], { body: ctx.body, items });
+      return out.reject != null
+        ? { value: out.items, dropped: out.dropped, reject: out.reject }
+        : { value: out.items, dropped: out.dropped };
+    },
+  },
   reject: {
-    label: "Reject the answer if…", accepts: ["list"],
+    label: "Reject the answer if…", accepts: ["list"], hidden: true,
     description: "Throws the whole reply away when a rule below matches it.",
     options: [{ key: "rules", label: "Rules", type: "rules", scope: "answer" }],
     defaults: () => ({ type: "reject", rules: [] }),
@@ -248,7 +492,7 @@ const LIST_MODIFIERS                                       = {
     apply: items => items.map(tidy).filter(Boolean),
   },
   drop: {
-    label: "Drop items", accepts: ["list"],
+    label: "Drop items", accepts: ["list"], hidden: true,
     description: "Removes each item a rule below matches.",
     options: [{ key: "rules", label: "Rules", type: "rules", scope: "item" }],
     defaults: () => ({ type: "drop", rules: [] }),

package/lab/metrics/builtin.mjs

@@ -135,16 +135,54 @@ function verdictOf(said        )
   if (read.error || !read.value || typeof read.value !== "object") throw new Error(`the grader's reply could not be read: ${short(said)}`);
   return read.value                           ;
 }
-const ask = async (ctx           , prompt        )                  => {
+/** The grader's verdict on [prompt], its reply held to [schema] where its API
+    can hold one. A reply that still cannot be read is asked for once more;
+    a second, or a grader that failed, is the grader's error -- said as one,
+    so it is not read as the reply failing. [ok] says whether a verdict read
+    is one this metric can use. */
+async function graded(ctx           , prompt        , schema                         ,
+                      ok                                               = () => true)                                        {
   if (!ctx.ask) throw new Error("a model-graded metric needs a grader, and this run has none");
-  return ctx.ask(prompt);
-};
+  let last = "";
+  for (let attempt = 0; attempt < 2; attempt++) {
+    let said        ;
+    try { said = await ctx.ask(prompt, schema); } catch (e) { throw new Error(`grader error: ${e instanceof Error ? e.message : String(e)}`); }
+    try {
+      const v = verdictOf(said);
+      if (ok(v)) return v;
+      last = `the grader said ${short(JSON.stringify(v))}`;
+    } catch (e) { last = e instanceof Error ? e.message : String(e); }
+  }
+  throw new Error(`grader error: ${last}`);
+}
 const gradedPass = (v                              , threshold        )                => {
   const score = typeof v.score === "number" ? Math.max(0, Math.min(1, v.score)) : v.pass === true ? 1 : 0;
   const pass = typeof v.pass === "boolean" ? v.pass && score >= threshold : score >= threshold;
   return { pass, score, reason: str(v.reason) || (pass ? "the grader passed it" : "the grader failed it") };
 };
+/** The shape a pass-or-fail grader answers in, for its API to hold it to. */
+const VERDICT_SCHEMA = { type: "object", additionalProperties: false, required: ["pass", "score", "reason"],
+  properties: { pass: { type: "boolean" }, score: { type: "number" }, reason: { type: "string" } } };
 const REPLY = 'Reply with one JSON object and nothing else: {"pass": true or false, "score": a number from 0 to 1, "reason": "one sentence"}.';
+/** What the grader is told of the task: what the target was asked, and the
+    guidance the metric adds, where there is either. */
+const taskOf = (input             , guidance         )         =>
+  (input.asked?.trim() ? `The task, as it was asked:\n${input.asked.trim()}\n\n` : "")
+  + (str(guidance).trim() ? `Guidance:\n${str(guidance).trim()}\n\n` : "");
+
+/** A grader's three verdicts on a reply set beside the recorded one; any of
+    them may be what a metric expects. */
+const JUDGEMENTS = [{ value: "better", label: "Better" }, { value: "same", label: "Same" }, { value: "worse", label: "Worse" }];
+const JUDGEMENT_SCORE                         = { better: 1, same: 0.5, worse: 0 };
+/** What a judgement metric expects: the verdicts set, or -- set before
+    Expected was offered -- the one rule there was, worse fails. */
+const expectedOf = (m        )           =>
+  (Array.isArray(m.expected) ? m.expected.map(String) : ["better", "same"]);
+/** The expected verdicts as a reader says them: "better or same". */
+const said = (values                   )         => {
+  const words = values.map((v) => v.toLowerCase());
+  return words.length < 2 ? words.join("") : `${words.slice(0, -1).join(", ")} or ${words.at(-1)}`;
+};
 
 // The families the Add metric picker lists the metrics under: what each reads.
 const TEXT = "The reply's text", RESULT = "The result", RECORDED = "Recorded reply", GRADED = "Model-graded";
@@ -342,6 +380,7 @@ const metrics                              = {
   // reply compared with the reply it is says nothing (docs/workflow-sources.md).
   "same-as-recorded": {
     family: RECORDED,
+    parsedOnly: true,
     label: "Same as recorded",
     description: "Passes when the reply is what production recorded for the same item.",
     options: [],
@@ -356,6 +395,7 @@ const metrics                              = {
   },
   "fields-equal-recorded": {
     family: RECORDED,
+    parsedOnly: true,
     label: "Fields as recorded",
     description: "Passes when the fields listed are what the recorded reply had.",
     options: [{ key: "fields", label: "Fields", type: "textarea" }],
@@ -374,6 +414,7 @@ const metrics                              = {
   },
   "same-parse-as-recorded": {
     family: RECORDED,
+    parsedOnly: true,
     label: "Parses as recorded",
     description: "Passes when the reply parses as JSON exactly when the recorded reply did.",
     options: [],
@@ -390,43 +431,58 @@ const metrics                              = {
     label: "Rubric",
     description: "Asks the grader whether the reply meets the rubric.",
     graded: true,
-    options: [{ key: "rubric", label: "Rubric", type: "textarea" }, { key: "threshold", label: "Pass at", type: "number" }],
+    parsedOnly: true,
+    options: [{ key: "rubric", label: "Rubric", type: "textarea" }, { key: "threshold", label: "Expected: score at least", type: "number" }],
     defaults: () => ({ rubric: "", threshold: 0.5 }),
     validate: (m, at, bad) => needs(m, ["rubric"], at, bad),
-    score: async (input, m, ctx) => gradedPass(verdictOf(await ask(ctx,
-      `You are grading an output against a rubric.\n\nRubric:\n${str(m.rubric)}\n\nOutput:\n${input.text}\n\n${REPLY}`)),
-      num(m.threshold) ?? 0.5),
+    score: async (input, m, ctx) => gradedPass(await graded(ctx,
+      `You are grading an output against a rubric.\n\n${taskOf(input, "")}Rubric:\n${str(m.rubric)}\n\nOutput:\n${input.text}\n\n${REPLY}`,
+      VERDICT_SCHEMA), num(m.threshold) ?? 0.5),
   },
   factuality: {
     family: GRADED,
     label: "Factual",
     description: "Asks the grader whether the reply agrees with the reference, or the recorded reply.",
     graded: true,
-    options: [{ key: "reference", label: "Reference", type: "textarea" }, { key: "threshold", label: "Pass at", type: "number" }],
+    parsedOnly: true,
+    options: [{ key: "reference", label: "Reference", type: "textarea" }, { key: "threshold", label: "Expected: score at least", type: "number" }],
     defaults: () => ({ reference: "", threshold: 0.5 }),
     // A blank reference is the recorded reply.
-    score: async (input, m, ctx) => gradedPass(verdictOf(await ask(ctx,
+    score: async (input, m, ctx) => gradedPass(await graded(ctx,
       `You are checking an output for factual consistency with a reference. Differences in wording or detail are `
-      + `fine; a contradiction, or a fact the reference does not support, is not.\n\nReference:\n`
-      + `${str(m.reference).trim() || recorded(input)}\n\nOutput:\n${input.text}\n\n${REPLY}`)), num(m.threshold) ?? 0.5),
+      + `fine; a contradiction, or a fact the reference does not support, is not.\n\n${taskOf(input, "")}Reference:\n`
+      + `${str(m.reference).trim() || recorded(input)}\n\nOutput:\n${input.text}\n\n${REPLY}`, VERDICT_SCHEMA), num(m.threshold) ?? 0.5),
   },
   "judge-vs-production": {
     family: GRADED,
     label: "Judged against recorded",
     description: "Asks the grader whether the reply is better than, the same as or worse than the recorded one.",
     graded: true,
-    options: [{ key: "task", label: "Task", type: "textarea" }],
-    defaults: () => ({ task: "" }),
+    parsedOnly: true,
+    expects: true,
+    options: [{ key: "expected", label: "Expected", type: "multi", choices: JUDGEMENTS },
+              { key: "task", label: "Additional guidance (optional)", type: "textarea" }],
+    defaults: () => ({ expected: ["better", "same"], task: "" }),
+    validate: (m, at, bad) => {
+      const known = JUDGEMENTS.map((j) => j.value);
+      if (m.expected !== undefined && (!Array.isArray(m.expected) || !m.expected.length
+          || m.expected.some((v) => !known.includes(String(v))))) {
+        bad.push(`${at}: Expected needs at least one of ${said(known)}`);
+      }
+    },
     score: async (input, m, ctx) => {
-      const v = verdictOf(await ask(ctx,
+      if (ctx.recordedTarget) return null;
+      const v = await graded(ctx,
         `Two outputs answer the same task. Say whether the NEW one is better than, the same as, or worse than the `
-        + `RECORDED one.${str(m.task).trim() ? `\n\nTask:\n${str(m.task)}` : ""}\n\nRECORDED:\n${recorded(input)}\n\n`
+        + `RECORDED one.\n\n${taskOf(input, m.task)}RECORDED:\n${recorded(input)}\n\n`
         + `NEW:\n${input.text}\n\nReply with one JSON object and nothing else: {"verdict": "better" or "same" or "worse", `
-        + `"reason": "one sentence"}.`));
-      const verdict = str(v.verdict).toLowerCase();
-      if (!["better", "same", "worse"].includes(verdict)) throw new Error(`the grader said ${short(JSON.stringify(v))}`);
-      return { pass: verdict !== "worse", score: verdict === "better" ? 1 : verdict === "same" ? 0.5 : 0,
-               reason: `${verdict}: ${str(v.reason)}` };
+        + `"reason": "one sentence"}.`,
+        { type: "object", additionalProperties: false, required: ["verdict", "reason"],
+          properties: { verdict: { type: "string", enum: JUDGEMENTS.map((j) => j.value) }, reason: { type: "string" } } },
+        (r) => str(r.verdict).toLowerCase() in JUDGEMENT_SCORE);
+      const verdict = str(v.verdict).toLowerCase(), expected = expectedOf(m);
+      return { pass: expected.includes(verdict), score: JUDGEMENT_SCORE[verdict] ,
+               reason: `${verdict} (expected ${said(expected)}): ${str(v.reason)}` };
     },
   },
 };

package/lab/run-evals.js

@@ -127,6 +127,10 @@ const USAGE = `Usage: node run-evals.js [options]
                       grades against each one its evals link (§17), so a run
                       linking several groups is handed them here instead of the
                       one --dataset. For --run and --rescore; not with --dataset.
+  --filter-sets <file> the filter-set bodies a --run kept, by "<id>@<n>" (§18):
+                      a Library set a job's Responses links cleans and gates
+                      its reply by. A private set carries its body in the
+                      document, so only linked sets are here. For --run.
   --source <dir>      the same, named the way a run names it: the directory a
                       Source is stored at. --samples and --source are one
                       flag by two names; both together are refused.
@@ -208,7 +212,7 @@ function broken(msg) {
 
 function parseArgs(argv) {
   const o = { model: "", url: "", prompt: null, pipeline: "", samples: "", dataset: "",
-              groups: "", replies: "", json: "", source: "", files: "", item: "", resume: "",
+              groups: "", filterSets: "", replies: "", json: "", source: "", files: "", item: "", resume: "",
               timeout: null, progress: false, cancelFile: "",
               run: "", progressFile: "", resultsFile: "", from: null, only: null,
               rescore: false, tokens: "", plugins: "", minPass: null, junit: "" };
@@ -230,6 +234,7 @@ function parseArgs(argv) {
       case "--samples": o.samples = value(); break;
       case "--dataset": o.dataset = value(); break;
       case "--groups": o.groups = value(); break;
+      case "--filter-sets": o.filterSets = value(); break;
       case "--replies": o.replies = value(); break;
       case "--source": o.source = value(); break;
       case "--files": o.files = value(); break;
@@ -395,6 +400,30 @@ function gradingContext(o) {
   return { single, bodies: null, primary: () => single, resolve: () => single ?? undefined };
 }
 
+// The filter-set bodies a run kept, as --filter-sets hands them (§18): a JSON
+// object of `<id>@<n>` to body -- a Library set's id and the version the run
+// cleaned and gated with -- each read as today's (upgradeFilterSetBody). A
+// private (`own`) set carries its body in the document, so only Library links
+// are here. Returns a resolver a job's filters step reads a link's body by,
+// mirroring the server's filter_set_ref / group_key.
+function filterContext(o) {
+  if (!o.filterSets) return () => undefined;
+  let raw;
+  try { raw = JSON.parse(fs.readFileSync(o.filterSets, "utf8")); }
+  catch (e) { broken(`${o.filterSets}: ${e.message}`); }
+  if (!isObject(raw)) broken(`${o.filterSets}: the kept filter sets are a JSON object of "<id>@<n>": body`);
+  const bodies = new Map();
+  for (const [key, body] of Object.entries(raw)) {
+    const at = `${o.filterSets} [${key}]`;
+    const got = core.upgradeFilterSetBody(body);
+    const bad = core.filterSetBodyProblems(got, at);
+    if (bad.length) broken(bad[0]);
+    bodies.set(key, got);
+  }
+  const key = (ref) => (ref && Number.isInteger(ref.n) ? `${ref.id}@${ref.n}` : `${ref && ref.id}`);
+  return (ref) => bodies.get(key(ref)) ?? bodies.get(String(ref && ref.id)) ?? undefined;
+}
+
 // The cases a graded run is scored against, by the item each names: exactly,
 // as the Source names it. Built from the one group a single-group run names,
 // for an eval type of its own that reads a case (a plugin's); a `group` eval
@@ -605,8 +634,12 @@ function graderFor(run) {
       if (!c) return undefined;
       if (!made.has(ref.id)) {
         const link = reach(c, core.keyVar(ref.id, c.slug));
-        made.set(ref.id, async prompt => {
-          const r = await ask(link, { id: ref.id, ...c }, prompt, null);
+        made.set(ref.id, async (prompt, schema) => {
+          // Held to the metric's schema where the grader's API can hold a
+          // reply to one; a model or server that refuses the schema is asked
+          // again without it, the prompt alone saying the shape.
+          let r = await ask(link, { id: ref.id, ...c }, prompt, null, schema);
+          if (r.error && schema) r = await ask(link, { id: ref.id, ...c }, prompt, null);
           if (r.error) throw new Error(`the grader ${c.name || ref.id} failed: ${r.error}`);
           return r.raw ?? "";
         });
@@ -660,13 +693,13 @@ async function send(to, c, step, cell, record) {
   }
 }
 
-async function ask(to, c, prompt, dataUrl) {
+async function ask(to, c, prompt, dataUrl, schema = null) {
   const t0 = Date.now();
   const said = more => ({ ms: Date.now() - t0, conn: to.name, ...more });
   try {
     // The connection's type builds the whole request: the URL, the headers
     // (Anthropic's key travels in x-api-key), and the body.
-    const req = core.connectionRequest(c, prompt, dataUrl, to.base, to.key);
+    const req = core.connectionRequest(c, prompt, dataUrl, to.base, to.key, schema);
     const r = await fetch(req.url, {
       method: "POST",
       headers: { "Content-Type": "application/json", ...req.headers },
@@ -896,9 +929,11 @@ async function runSnapshot(o) {
 
   // Each scenario once: its stages, the jobs' token sets, and a transport
   // per stage to the profile that stage resolves to, keyed by that
-  // profile's id.
+  // profile's id. Its stages carry each linked or private filter set as a
+  // modifier that cleans and gates the reply where the link sits (#339).
+  const resolveFilters = filterContext(o);
   const plans = core.targetsOf(run).map((_, i) => {
-    const { stages, tokens, connections, calls, cells } = core.stagesFor(run, i);
+    const { stages, tokens, connections, calls, cells } = core.stagesFor(run, i, resolveFilters);
     const links = connections.map(c => reach(c, core.keyVar(c.id, c.slug)));
     // The Recorded target replays the recorded reply, so a metric that compares
     // with the recorded reply has nothing to say of it (n/a).