evals-lab 0.8.0 → 0.9.0

package/lab/kinds/list.mjs

@@ -125,21 +125,26 @@ export function dropBy(rules                     , item        )
     `itemCount`, and `share` of the items `items` matches. Every one given must
     hold; a rule that states none never fires. */
 export function rejectBy(rules                     , body        , items                   )                {
-  for (const r of rules) {
-    const m = r.match || {};
-    const conds            = [];
-    if (m.blank === true) conds.push(!body.trim());
-    if (m.itemCount && typeof m.itemCount === "object") conds.push(inRange(m.itemCount         )(items.length));
-    if (m.share && typeof m.share === "object") {
-      const test = matcher(m.items                           );
-      // A share with nothing to measure cannot fire.
-      conds.push(!!test && inRange(m.share         )(items.length ? items.filter(test).length / items.length : 0));
-    }
-    if (conds.length && conds.every(Boolean)) return r.id;
-  }
+  for (const r of rules) if (rejectHolds(r.match || {}, body, items)) return r.id;
   return null;
 }
 
+/** Whether an answer whose text is [body] and items are [items] meets the
+    reject conditions [m]: `blank`, `itemCount` and the `share` of `items` the
+    items match. Every one given must hold; one that states none never fires.
+    The condition half of rejectBy, shared with the reject-reply filter. */
+function rejectHolds(m                         , body        , items                   )          {
+  const conds            = [];
+  if (m.blank === true) conds.push(!body.trim());
+  if (m.itemCount && typeof m.itemCount === "object") conds.push(inRange(m.itemCount         )(items.length));
+  if (m.share && typeof m.share === "object") {
+    const test = matcher(m.items                           );
+    // A share with nothing to measure cannot fire.
+    conds.push(!!test && inRange(m.share         )(items.length ? items.filter(test).length / items.length : 0));
+  }
+  return conds.length > 0 && conds.every(Boolean);
+}
+
 /** Why [rules] are not a rule list [scope] takes, one sentence each. */
 export function itemRulesProblems(rules         , scope                   , at        )           {
   if (!Array.isArray(rules)) return [`${at}: rules has to be a list`];
@@ -184,6 +189,226 @@ export function tryExamples(rule          , scope                   )
   ];
 }
 
+// ---- filters: the generic reply filter (issue #335) ----------------------------
+//
+// A Filter is the reusable unit #334 introduces: it supersedes the inline
+// ItemRule a Drop items or Reject the answer modifier carries today. Where a
+// rule was a bare matcher with examples, a filter has a base TYPE from a
+// registry (not an enum) -- "drop-item" cleans each item of a list reply,
+// "reject-reply" gates the whole answer -- generic conditions reusing the
+// matcher above (ANDed), Options, and Tests that are its own checks. The two
+// base types evaluate here, beside the matcher and rejectHolds they are built
+// from; the Library-stored filter-set body, the pipeline link that carries
+// one, and their version live in evals-core.ts, as DatasetBody and the
+// eval-group link do. Nothing here replaces the Drop items / Reject modifiers:
+// Runs wires a filter set into Responses and migrates the inline rules in a
+// later phase (#339), so both base types and the ItemRule shape stand.
+
+/** How a filter test splits its sample reply into items. "none" keeps the
+    reply whole, for a reject-reply filter that reads the text. */
+export const FILTER_SEPARATORS = ["comma", "lines", "semicolon", "none"]         ;
+                                                                 
+
+/** A sample reply and what the filter should leave of it: the editor splits
+    the sample by `separator`, applies the filter, and shows whether the result
+    reads as `expectedOutput`. A drop-item filter's expected output is the
+    items it keeps; a reject-reply filter's is blank when the reply should be
+    rejected, or left non-blank when it should pass. */
+                             
+                             
+                      
+                         
+ 
+
+/** A filter's Options: keep going when it errors (rather than failing the
+    run), and how many passes it makes over the reply. */
+                                
+                          
+                 
+ 
+
+/** A filter (issue #335): a base TYPE fixed at creation (a FILTER_TYPES id),
+    generic conditions in `match` (the matcher's keys for its type, ANDed),
+    Options and Tests. `id` is its name, as an ItemRule's was. */
+                         
+             
+               
+               
+                                 
+                         
+                      
+ 
+
+/** A reply a filter works over: the text it came as, and the items parsed
+    from it (as the List kind parses a run's reply). */
+                                                              
+
+/** What applying a filter or a set left: the items kept, what was dropped and
+    by which filter, and -- the first reject-reply filter that fired -- the
+    reply's rejection (null when none did). */
+                                                                                             
+
+/** A base filter type (#334's decision: a registry, not an enum). A reader
+    asks the entry by `filter.type`; nothing branches on the id. */
+                                  
+                                                
+                
+                                                                               
+                           
+                                                                      
+                                                            
+                                
+                                                                            
+                                                                 
+                                                           
+ 
+
+/** The lab's base filter types. Generic code asks this registry by
+    `filter.type`; it never names "drop-item" or "reject-reply". */
+export const FILTER_TYPES                                  = {
+  "drop-item": {
+    label: "Drop item", scope: "item", conditions: MATCHER_KEYS,
+    apply(filter, reply) {
+      const test = matcher(filter.match);
+      if (!test) return { items: reply.items, dropped: [], reject: null };
+      const items           = [], dropped            = [];
+      for (const t of reply.items) {
+        if (test(t)) dropped.push({ item: t, by: filter.id });
+        else items.push(t);
+      }
+      return { items, dropped, reject: null };
+    },
+  },
+  "reject-reply": {
+    label: "Reject reply", scope: "answer", conditions: ["blank", "itemCount", "share", "items"],
+    apply(filter, reply) {
+      return { items: reply.items, dropped: [], reject: rejectHolds(filter.match, reply.body, reply.items) ? filter.id : null };
+    },
+  },
+};
+
+/** Apply one filter to [reply], honouring its Options: `repeat` passes over
+    the reply (a reject-reply filter stops at the first that fires), and
+    `ignoreFailures` leaves the reply as it was if the filter errors -- a
+    condition that cannot be read -- rather than failing the run. Pure. */
+export function applyFilter(filter        , reply             )                {
+  const entry = FILTER_TYPES[filter.type];
+  if (!entry) return { items: reply.items, dropped: [], reject: null };
+  const passes = Number.isInteger(filter.options?.repeat) && filter.options.repeat > 0 ? filter.options.repeat : 1;
+  let items = reply.items;
+  const dropped            = [];
+  let reject                = null;
+  for (let pass = 0; pass < passes && reject == null; pass++) {
+    try {
+      const out = entry.apply(filter, { body: reply.body, items });
+      items = out.items;
+      dropped.push(...out.dropped);
+      reject = out.reject;
+    } catch (e) {
+      // Ignore failures: leave the reply as this pass found it; else the run
+      // records the fault.
+      if (!filter.options?.ignoreFailures) throw e;
+      break;
+    }
+  }
+  return { items, dropped, reject };
+}
+
+/** Apply a filter set's filters to [reply], in order: each drop-item filter
+    narrows the items the next reads, and the first reject-reply filter that
+    fires rejects the answer and stops the rest. Pure. */
+export function applyFilterSet(filters                   , reply             )                {
+  let items = reply.items;
+  const dropped            = [];
+  for (const filter of filters) {
+    const out = applyFilter(filter, { body: reply.body, items });
+    items = out.items;
+    dropped.push(...out.dropped);
+    if (out.reject != null) return { items, dropped, reject: out.reject };
+  }
+  return { items, dropped, reject: null };
+}
+
+const SEPARATOR_RE                                         = { comma: /,/, lines: /\n/, semicolon: /;/, none: null };
+
+/** [sample] split into items by [separator]: each trimmed, none blank. "none"
+    keeps the reply whole as one item. */
+export function splitSample(sample        , separator                 )           {
+  const re = SEPARATOR_RE[separator];
+  const parts = re ? String(sample ?? "").split(re) : [String(sample ?? "")];
+  return parts.map(s => s.trim()).filter(Boolean);
+}
+
+/** Each of [filter]'s tests, tried against the filter alone: the sample split
+    by the test's separator, the filter applied, and the result compared with
+    the expected output. A drop-item test passes when the items kept read as
+    the expected items; a reject-reply test passes when the reply is rejected
+    exactly if the expected output is blank. Supersedes tryExamples. */
+export function tryFilter(filter        )                                                   {
+  const scope = FILTER_TYPES[filter.type]?.scope ?? "item";
+  return (filter.tests || []).map(test => {
+    const out = applyFilter(filter, { body: test.sampleInput, items: splitSample(test.sampleInput, test.separator) });
+    if (scope === "answer") {
+      const rejected = out.reject != null;
+      return { test, ok: rejected === !test.expectedOutput.trim(), got: rejected ? "rejected" : out.items.join(", ") };
+    }
+    const want = splitSample(test.expectedOutput, test.separator);
+    const ok = out.items.length === want.length && out.items.every((t, i) => t === want[i]);
+    return { test, ok, got: out.items.join(", ") };
+  });
+}
+
+/** Why [filters] are not a filter list the lab reads, one sentence each: each
+    has a name, no name twice, a base type it has, whole-number repeat of 1 or
+    more, a boolean ignore-failures, and a matcher its type's scope takes
+    (itemRulesProblems). */
+export function filterProblems(filters         , at        )           {
+  if (!Array.isArray(filters)) return [`${at}: filters has to be a list`];
+  const bad           = [], seen = new Set        ();
+  for (const f of filters            ) {
+    const id = f && typeof f.id === "string" && f.id.trim() ? f.id : null;
+    if (!id) { bad.push(`${at}: a filter has no name`); continue; }
+    if (seen.has(id)) bad.push(`${at}: ${id} is named twice`);
+    seen.add(id);
+    const entry = typeof f.type === "string" ? FILTER_TYPES[f.type] : undefined;
+    if (!entry) { bad.push(`${at}: ${id} has no base type`); continue; }
+    const o = f.options;
+    if (!o || typeof o !== "object" || Array.isArray(o)) bad.push(`${at}: ${id} has no options`);
+    else {
+      if (!Number.isInteger(o.repeat) || o.repeat < 1) bad.push(`${at}: ${id}'s repeat has to be a whole number of 1 or more`);
+      if (typeof o.ignoreFailures !== "boolean") bad.push(`${at}: ${id}'s ignore failures has to be true or false`);
+    }
+    bad.push(...itemRulesProblems([{ id, match: f.match }], entry.scope, at));
+  }
+  return bad;
+}
+
+/** The filter an inline ItemRule becomes (issue #335): its name, why and
+    matcher kept as they are, its base [type] the one the modifier carrying it
+    fixed ("drop-item" for Drop items, "reject-reply" for Reject the answer),
+    its Options the defaults, and its examples its tests -- a caught example
+    becomes a sample the filter should empty (drop-item) or reject
+    (reject-reply), a kept example one it should leave. Pure: the match is
+    carried unchanged, so the filter cleans a reply exactly as the rule did.
+    The only reader of the old shape, for Runs to migrate its inline rules
+    (#339). */
+export function filterFromItemRule(rule          , type        )         {
+  const scope = FILTER_TYPES[type]?.scope ?? "item";
+  const ex = rule.examples || {};
+  const caught = scope === "item" ? ex.drops || [] : ex.rejects || [];
+  const tests               = [
+    ...caught.map((s)             => ({ separator: "comma", sampleInput: s, expectedOutput: "" })),
+    ...(ex.keeps || []).map((s)             => ({ separator: "comma", sampleInput: s, expectedOutput: s })),
+  ];
+  return {
+    id: rule.id, type,
+    ...(rule.why != null ? { why: rule.why } : {}),
+    match: rule.match ? { ...rule.match } : {},
+    options: { ignoreFailures: false, repeat: 1 },
+    tests,
+  };
+}
+
 // ---- the modifiers -------------------------------------------------------------
 
 /** What an item's numbering looks like: "[1] suv", "(2) car", "3. dirt",
@@ -225,9 +450,28 @@ export function applyCase(item        , mode        )         {
   }
 }
 
+/** The id of the synthetic modifier a linked or private filter set applies
+    under at run time (#339): it is never in a document or the Add menu -- a
+    job carries a `filters` step, which `stageModifiers` turns into one of
+    these where the step sits, so a set cleans and gates a list reply exactly
+    as the Drop items / Reject modifiers it replaced did, in the same order. */
+export const FILTER_MODIFIER = "filterSet";
+
 const LIST_MODIFIERS                                       = {
+  // A filter set's filters, applied where its link sits: the run's only reader
+  // of a resolved set, hidden from the Add menu (no options/defaults) because
+  // a set is linked in Responses, not added as a modifier.
+  [FILTER_MODIFIER]: {
+    accepts: ["list"], hidden: true,
+    apply(items, m, ctx) {
+      const out = applyFilterSet((m.filters            ) || [], { body: ctx.body, items });
+      return out.reject != null
+        ? { value: out.items, dropped: out.dropped, reject: out.reject }
+        : { value: out.items, dropped: out.dropped };
+    },
+  },
   reject: {
-    label: "Reject the answer if…", accepts: ["list"],
+    label: "Reject the answer if…", accepts: ["list"], hidden: true,
     description: "Throws the whole reply away when a rule below matches it.",
     options: [{ key: "rules", label: "Rules", type: "rules", scope: "answer" }],
     defaults: () => ({ type: "reject", rules: [] }),
@@ -248,7 +492,7 @@ const LIST_MODIFIERS                                       = {
     apply: items => items.map(tidy).filter(Boolean),
   },
   drop: {
-    label: "Drop items", accepts: ["list"],
+    label: "Drop items", accepts: ["list"], hidden: true,
     description: "Removes each item a rule below matches.",
     options: [{ key: "rules", label: "Rules", type: "rules", scope: "item" }],
     defaults: () => ({ type: "drop", rules: [] }),

package/lab/run-evals.js

@@ -127,6 +127,10 @@ const USAGE = `Usage: node run-evals.js [options]
                       grades against each one its evals link (§17), so a run
                       linking several groups is handed them here instead of the
                       one --dataset. For --run and --rescore; not with --dataset.
+  --filter-sets <file> the filter-set bodies a --run kept, by "<id>@<n>" (§18):
+                      a Library set a job's Responses links cleans and gates
+                      its reply by. A private set carries its body in the
+                      document, so only linked sets are here. For --run.
   --source <dir>      the same, named the way a run names it: the directory a
                       Source is stored at. --samples and --source are one
                       flag by two names; both together are refused.
@@ -208,7 +212,7 @@ function broken(msg) {
 
 function parseArgs(argv) {
   const o = { model: "", url: "", prompt: null, pipeline: "", samples: "", dataset: "",
-              groups: "", replies: "", json: "", source: "", files: "", item: "", resume: "",
+              groups: "", filterSets: "", replies: "", json: "", source: "", files: "", item: "", resume: "",
               timeout: null, progress: false, cancelFile: "",
               run: "", progressFile: "", resultsFile: "", from: null, only: null,
               rescore: false, tokens: "", plugins: "", minPass: null, junit: "" };
@@ -230,6 +234,7 @@ function parseArgs(argv) {
       case "--samples": o.samples = value(); break;
       case "--dataset": o.dataset = value(); break;
       case "--groups": o.groups = value(); break;
+      case "--filter-sets": o.filterSets = value(); break;
       case "--replies": o.replies = value(); break;
       case "--source": o.source = value(); break;
       case "--files": o.files = value(); break;
@@ -395,6 +400,30 @@ function gradingContext(o) {
   return { single, bodies: null, primary: () => single, resolve: () => single ?? undefined };
 }
 
+// The filter-set bodies a run kept, as --filter-sets hands them (§18): a JSON
+// object of `<id>@<n>` to body -- a Library set's id and the version the run
+// cleaned and gated with -- each read as today's (upgradeFilterSetBody). A
+// private (`own`) set carries its body in the document, so only Library links
+// are here. Returns a resolver a job's filters step reads a link's body by,
+// mirroring the server's filter_set_ref / group_key.
+function filterContext(o) {
+  if (!o.filterSets) return () => undefined;
+  let raw;
+  try { raw = JSON.parse(fs.readFileSync(o.filterSets, "utf8")); }
+  catch (e) { broken(`${o.filterSets}: ${e.message}`); }
+  if (!isObject(raw)) broken(`${o.filterSets}: the kept filter sets are a JSON object of "<id>@<n>": body`);
+  const bodies = new Map();
+  for (const [key, body] of Object.entries(raw)) {
+    const at = `${o.filterSets} [${key}]`;
+    const got = core.upgradeFilterSetBody(body);
+    const bad = core.filterSetBodyProblems(got, at);
+    if (bad.length) broken(bad[0]);
+    bodies.set(key, got);
+  }
+  const key = (ref) => (ref && Number.isInteger(ref.n) ? `${ref.id}@${ref.n}` : `${ref && ref.id}`);
+  return (ref) => bodies.get(key(ref)) ?? bodies.get(String(ref && ref.id)) ?? undefined;
+}
+
 // The cases a graded run is scored against, by the item each names: exactly,
 // as the Source names it. Built from the one group a single-group run names,
 // for an eval type of its own that reads a case (a plugin's); a `group` eval
@@ -900,9 +929,11 @@ async function runSnapshot(o) {
 
   // Each scenario once: its stages, the jobs' token sets, and a transport
   // per stage to the profile that stage resolves to, keyed by that
-  // profile's id.
+  // profile's id. Its stages carry each linked or private filter set as a
+  // modifier that cleans and gates the reply where the link sits (#339).
+  const resolveFilters = filterContext(o);
   const plans = core.targetsOf(run).map((_, i) => {
-    const { stages, tokens, connections, calls, cells } = core.stagesFor(run, i);
+    const { stages, tokens, connections, calls, cells } = core.stagesFor(run, i, resolveFilters);
     const links = connections.map(c => reach(c, core.keyVar(c.id, c.slug)));
     // The Recorded target replays the recorded reply, so a metric that compares
     // with the recorded reply has nothing to say of it (n/a).