evals-lab 0.1.2

package/lab/kinds/list.mjs

@@ -0,0 +1,411 @@
+// A reply that is a list of items: the `list` output kind and the modifiers
+// that clean one, as entries in the lab's registries (docs/pipeline-model.md
+// §2; docs/design/generic-lab.md §2).
+//
+// Nothing here is about any one app. A list is parsed from the reply -- CSV,
+// one item a line, or a JSON array -- and every cleaning step is a modifier a
+// pipeline chooses, in the order it chooses: reject the answer, tidy the
+// items, drop items by rule, drop items that repeat the instruction, remove
+// duplicates, keep the first n, change case. A particular app's parser is a
+// particular configuration of these, stated in its pipeline -- never code
+// here. `legacyTagsOut` is the one configuration this file names, because
+// it is what a pipeline of the retired `tags` kind is upgraded to.
+//
+// Rules are data a modifier carries: an item matcher (terms, a regex, the
+// item's length, its words, its whitespace-separated tokens, its share of
+// digits), a why, and examples that are the rule's own tests.
+                                                                                                            
+
+// ---- text helpers ------------------------------------------------------------
+
+/** Letters and digits, lower-cased: what "a word" means to a matcher. */
+export const words = (s         )           => String(s ?? "").toLowerCase().match(/[\p{L}\p{N}]+/gu) || [];
+
+/** Both ends, those characters only: a regex over the whole value would take
+    them out of the middle as well. */
+export function trimEnds(s        , chars        )         {
+  let a = 0, b = s.length;
+  while (a < b && chars.includes(s[a] )) a++;
+  while (b > a && chars.includes(s[b - 1] )) b--;
+  return s.slice(a, b);
+}
+
+/** Whitespace collapsed, ends trimmed, case ignored: what "equal" means. */
+const normalise = (term         )         => String(term).toLowerCase().replace(/\s+/g, " ").trim();
+
+/** The reply with a thinking block and the preamble up to the first colon
+    taken off, or the whole reply when that would leave nothing. */
+export function withoutPreamble(reply         )         {
+  const after = String(reply ?? "").split("</think>").pop() ;
+  const i = after.indexOf(":");
+  const body = i < 0 ? after : after.slice(i + 1);
+  return body.trim() ? body : String(reply ?? "");
+}
+
+// ---- parsing a reply into items ------------------------------------------------
+
+export const PARSE_FORMATS = ["csv", "lines", "json"]         ;
+                                                         
+
+/** A reply as the text its items come from, and the items, each trimmed and
+    none blank. Nothing else is done to them: cleaning is the modifiers'. */
+export function splitItems(reply        , parse              = "csv", preamble = false)                                    {
+  const body = preamble ? withoutPreamble(reply) : String(reply ?? "");
+  if (parse === "json") {
+    const at = body.search(/[[{]/);
+    let items           = [];
+    try {
+      const doc = JSON.parse(at < 0 ? body : body.slice(at))           ;
+      const list = Array.isArray(doc) ? doc
+        : doc && typeof doc === "object" ? Object.values(doc).find(Array.isArray) : null;
+      items = (list || []).map((v         ) => (typeof v === "string" ? v : JSON.stringify(v)).trim()).filter(Boolean);
+    } catch { /* not JSON: no items, which a Reject rule can say */ }
+    return { body, items };
+  }
+  const sep = parse === "lines" ? /\n/ : /[,\n;]/;
+  return { body, items: body.split(sep).map(s => s.trim()).filter(Boolean) };
+}
+
+// ---- matching an item ----------------------------------------------------------
+
+/** A bound on a number: inclusive `min`/`max`, exclusive `above`/`below`. */
+                                                                              
+const inRange = (b       ) => (v        ) =>
+  (b.min == null || v >= b.min) && (b.max == null || v <= b.max)
+  && (b.above == null || v > b.above) && (b.below == null || v < b.below);
+
+/** The predicates an item matcher takes: every one given must hold. */
+export const MATCHER_KEYS = ["terms", "regex", "length", "words", "tokens", "digitShare"]         ;
+
+const regexCache = new Map                        ();
+function regexMatch(pattern        , item        )          {
+  let re = regexCache.get(pattern);
+  if (re === undefined) {
+    try { re = new RegExp("^(?:" + pattern + ")$", "i"); } catch { re = false; }
+    regexCache.set(pattern, re);
+  }
+  return !!re && re.test(item);
+}
+const digitShare = (s        )         => (s.length ? (s.match(/\d/g) || []).length / s.length : 0);
+
+/** An item matcher as one test, or null when it states nothing -- a rule that
+    states nothing never fires, so a malformed rule drops nothing on a guess. */
+export function matcher(m                                            )                                     {
+  if (!m || typeof m !== "object") return null;
+  const ps                                = [];
+  if (typeof m.regex === "string") { const r = m.regex; ps.push(t => regexMatch(r, t)); }
+  if (Array.isArray(m.terms)) { const set = new Set(m.terms.map(normalise)); ps.push(t => set.has(normalise(t))); }
+  if (m.length && typeof m.length === "object") { const b = inRange(m.length         ); ps.push(t => b(t.length)); }
+  if (m.words && typeof m.words === "object") { const b = inRange(m.words         ); ps.push(t => b(words(t).length)); }
+  if (m.tokens && typeof m.tokens === "object") { const b = inRange(m.tokens         ); ps.push(t => b(t.split(/\s+/).length)); }
+  if (m.digitShare && typeof m.digitShare === "object") { const b = inRange(m.digitShare         ); ps.push(t => b(digitShare(t))); }
+  return ps.length ? (t        ) => ps.every(p => p(t)) : null;
+}
+
+/** A rule a Drop items or a Reject the answer modifier carries. */
+                           
+             
+               
+                                  
+                                                                                              
+                                                                        
+ 
+
+/** The first rule that drops [item], or null. */
+export function dropBy(rules                     , item        )                {
+  for (const r of rules) {
+    const test = matcher(r.match);
+    if (test && test(item)) return r.id;
+  }
+  return null;
+}
+
+/** The first rule that rejects an answer whose text is [body] and items are
+    [items], or null. A rule's conditions: `blank` (the text is blank),
+    `itemCount`, and `share` of the items `items` matches. Every one given must
+    hold; a rule that states none never fires. */
+export function rejectBy(rules                     , body        , items                   )                {
+  for (const r of rules) {
+    const m = r.match || {};
+    const conds            = [];
+    if (m.blank === true) conds.push(!body.trim());
+    if (m.itemCount && typeof m.itemCount === "object") conds.push(inRange(m.itemCount         )(items.length));
+    if (m.share && typeof m.share === "object") {
+      const test = matcher(m.items                           );
+      // A share with nothing to measure cannot fire.
+      conds.push(!!test && inRange(m.share         )(items.length ? items.filter(test).length / items.length : 0));
+    }
+    if (conds.length && conds.every(Boolean)) return r.id;
+  }
+  return null;
+}
+
+/** Why [rules] are not a rule list [scope] takes, one sentence each. */
+export function itemRulesProblems(rules         , scope                   , at        )           {
+  if (!Array.isArray(rules)) return [`${at}: rules has to be a list`];
+  const bad           = [], seen = new Set        ();
+  const allowed = scope === "item" ? MATCHER_KEYS                      : ["blank", "itemCount", "share", "items"];
+  const rangeOk = (v         ) => !!v && typeof v === "object" && !Array.isArray(v)
+    && Object.entries(v).every(([k, n]) => ["min", "max", "above", "below"].includes(k) && typeof n === "number" && isFinite(n));
+  for (const r of rules              ) {
+    const id = r && typeof r.id === "string" && r.id.trim() ? r.id : null;
+    if (!id) { bad.push(`${at}: a rule has no name`); continue; }
+    if (seen.has(id)) bad.push(`${at}: ${id} is named twice`);
+    seen.add(id);
+    const m = r.match;
+    if (!m || typeof m !== "object" || Array.isArray(m)) { bad.push(`${at}: ${id} matches nothing`); continue; }
+    for (const [k, v] of Object.entries(m)) {
+      if (!allowed.includes(k)) { bad.push(`${at}: ${id} has "${k}", which is not a condition this rule takes`); continue; }
+      if (k === "terms" && !(Array.isArray(v) && v.every(x => typeof x === "string"))) bad.push(`${at}: ${id}'s terms have to be a list of words`);
+      if (k === "regex" && typeof v !== "string") bad.push(`${at}: ${id}'s regex has to be text`);
+      if (k === "regex" && typeof v === "string") { try { new RegExp(v); } catch { bad.push(`${at}: ${id}'s regex does not compile`); } }
+      if (k === "blank" && v !== true) bad.push(`${at}: ${id}'s blank can only be true`);
+      if (k === "items" && (itemRulesProblems([{ id, match: v                            }], "item", at).length)) bad.push(`${at}: ${id}'s items matcher is not one`);
+      if (["length", "words", "tokens", "digitShare", "itemCount", "share"].includes(k) && !rangeOk(v)) {
+        bad.push(`${at}: ${id}'s ${k} has to be a range of min, max, above and below`);
+      }
+    }
+    if (scope === "answer" && "share" in m && !("items" in m)) bad.push(`${at}: ${id} measures a share with nothing to count`);
+  }
+  return bad;
+}
+
+/** Each of [rule]'s examples, tried against the rule alone: a drop example is
+    dropped (or a reject example rejected), a keep example kept. */
+export function tryExamples(rule          , scope                   ) 
+                                                             {
+  const hits = (text        ) => (scope === "item"
+    ? dropBy([rule], text)
+    : (() => { const { body, items } = splitItems(text, "csv", true); return rejectBy([rule], body, items); })()) != null;
+  const ex = rule.examples || {};
+  return [
+    ...(scope === "item" ? ex.drops || [] : ex.rejects || []).map(text => ({ text, side: "drops"         , ok: hits(text) })),
+    ...(ex.keeps || []).map(text => ({ text, side: "keeps"         , ok: !hits(text) })),
+  ];
+}
+
+// ---- the modifiers -------------------------------------------------------------
+
+/** What an item's numbering looks like: "[1] suv", "(2) car", "3. dirt",
+    "4) trees", "5: sky" -- and not "1950s car" or "5.1 surround". */
+const NUMBERING = /^(?:\[\d+\]|\(\d+\)|\d+[.):](?!\d))\s*/;
+
+/** One item as Tidy items leaves it: a leading bullet, its numbering, stray
+    quotes and stops at the ends, and runs of whitespace gone. */
+export function tidy(item        )         {
+  let t = item.trim();
+  for (const p of ["-", "*", "•"]) if (t.startsWith(p)) t = t.slice(p.length);
+  t = t.trim().replace(NUMBERING, "");
+  t = trimEnds(t.trim(), ".\"'`");
+  return t.replace(/\s+/g, " ").trim();
+}
+
+/** The name a Remove duplicates or Drop prompt echoes drop is recorded under. */
+export const ECHO = "echoes the prompt";
+export const DUPLICATE = "already there";
+const pastN = (n        ) => `past ${n}`;
+
+                      
+const kept = (value       , dropped           ) => ({ value, dropped });
+
+/** The casings a list is shown in. Items are compared ignoring case, so this
+    is presentation and never a match rule. */
+export const CASE_MODES = ["LOWER", "UPPER", "NONE", "SENTENCE", "TITLE"];
+
+export function applyCase(item        , mode        )         {
+  const lower = item.toLowerCase();
+  const cap = (s        ) => s.charAt(0).toUpperCase() + s.slice(1);
+  switch (mode) {
+    case "LOWER": return lower;
+    case "UPPER": return item.toUpperCase();
+    case "NONE": return item;
+    case "SENTENCE": return cap(lower);
+    case "TITLE": return lower.split(" ").map(cap).join(" ");
+    default: return lower;
+  }
+}
+
+const LIST_MODIFIERS                                       = {
+  reject: {
+    label: "Reject the answer if…", accepts: ["list"],
+    description: "Throws the whole reply away when a rule below matches it.",
+    options: [{ key: "rules", label: "Rules", type: "rules", scope: "answer" }],
+    defaults: () => ({ type: "reject", rules: [] }),
+    validate(m, at, bad) { bad.push(...itemRulesProblems(m.rules, "answer", at)); },
+    // Measured on the items as parsed, before any tidying: "1. a red car on
+    // the road" is five pieces of text as the model wrote it.
+    apply(items, m, ctx) {
+      const why = rejectBy((m.rules              ) || [], ctx.body, items);
+      return why ? { value: items, reject: why } : items;
+    },
+  },
+  tidy: {
+    label: "Tidy items", accepts: ["list"],
+    description: "Cleans each item: bullets, numbering, stray quotes and full stops and extra spaces are removed, and empty items dropped.",
+    options: [],
+    defaults: () => ({ type: "tidy" }),
+    // A blank item is nothing at all, not one the list decided against.
+    apply: items => items.map(tidy).filter(Boolean),
+  },
+  drop: {
+    label: "Drop items", accepts: ["list"],
+    description: "Removes each item a rule below matches.",
+    options: [{ key: "rules", label: "Rules", type: "rules", scope: "item" }],
+    defaults: () => ({ type: "drop", rules: [] }),
+    validate(m, at, bad) { bad.push(...itemRulesProblems(m.rules, "item", at)); },
+    apply(items, m) {
+      const rules = (m.rules              ) || [];
+      const out        = [], dropped            = [];
+      for (const t of items) {
+        const by = dropBy(rules, t);
+        if (by) dropped.push({ item: t, by }); else out.push(t);
+      }
+      return kept(out, dropped);
+    },
+  },
+  "drop-echoes": {
+    label: "Drop prompt echoes", accepts: ["list"],
+    description: "Removes items made only of words from the prompt, which the model repeated rather than answered with.",
+    options: [],
+    defaults: () => ({ type: "drop-echoes" }),
+    // Every word of the item is in the stage's own instruction: "wellington
+    // region" is wholly the prompt's; an item that shares one ordinary word
+    // with it is not.
+    apply(items, _m, ctx) {
+      const said = ctx.instruction.trim() ? new Set(words(ctx.instruction)) : new Set        ();
+      const out        = [], dropped            = [];
+      for (const t of items) {
+        const w = words(t);
+        if (said.size && w.length && w.every(x => said.has(x))) dropped.push({ item: t, by: ECHO });
+        else out.push(t);
+      }
+      return kept(out, dropped);
+    },
+  },
+  dedupe: {
+    label: "Remove duplicates", accepts: ["list"],
+    description: "Keeps the first of items that differ only in case.",
+    options: [],
+    defaults: () => ({ type: "dedupe" }),
+    apply(items) {
+      const seen = new Set        (), out        = [], dropped            = [];
+      for (const t of items) {
+        const k = t.toLowerCase();
+        if (seen.has(k)) dropped.push({ item: t, by: DUPLICATE });
+        else { seen.add(k); out.push(t); }
+      }
+      return kept(out, dropped);
+    },
+  },
+  case: {
+    label: "Case", accepts: ["list"], modes: CASE_MODES,
+    description: "Changes each item to the case chosen.",
+    options: [{ key: "mode", label: "Mode", type: "select", choices: [
+      { value: "LOWER", label: "lower case" }, { value: "SENTENCE", label: "Sentence case" },
+      { value: "TITLE", label: "Title Case" }, { value: "UPPER", label: "UPPER CASE" },
+      { value: "NONE", label: "No change" }] }],
+    defaults: () => ({ type: "case", mode: "LOWER" }),
+    validate(m, at, bad) {
+      if (!CASE_MODES.includes(m.mode          )) bad.push(`${at} has to be one of ${CASE_MODES.join(", ")}`);
+      for (const k of Object.keys(m)) {
+        if (k !== "type" && k !== "mode") bad.push(`${at} has "${k}", which is not a pipeline field`);
+      }
+    },
+    apply: (items, m) => items.map(t => applyCase(t, m.mode          )),
+  },
+  "keep-first": {
+    label: "Keep first", accepts: ["list"],
+    description: "Keeps only the first items, as many as set.",
+    options: [{ key: "n", label: "Items", type: "number" }],
+    defaults: () => ({ type: "keep-first", n: 10 }),
+    validate(m, at, bad) {
+      if (!Number.isInteger(m.n) || (m.n          ) < 1) bad.push(`${at}: keep first needs a whole number of 1 or more`);
+    },
+    apply(items, m) {
+      const n = m.n          ;
+      return kept(items.slice(0, n), items.slice(n).map(t => ({ item: t, by: pastN(n) })));
+    },
+  },
+};
+
+// ---- the kind --------------------------------------------------------------------
+
+const LIST_KIND                         = {
+  label: "List", noun: "items", offers: ["items"],
+  description: "Splits the reply into items (by commas, lines or a JSON array) that tests and later jobs read one by one.",
+  settings: [
+    { key: "parse", label: "Parse as", type: "select", choices: [
+      { value: "csv", label: "CSV" }, { value: "lines", label: "One a line" }, { value: "json", label: "JSON array" }] },
+    { key: "preamble", label: "Ignore preamble", type: "checkbox" },
+  ],
+  settingDefaults: () => ({ parse: "csv", preamble: false }),
+  validateSettings(out, at, bad) {
+    if (out.parse != null && !PARSE_FORMATS.includes(out.parse               )) {
+      bad.push(`${at}: parse has to be one of ${PARSE_FORMATS.join(", ")}`);
+    }
+    if (out.preamble != null && typeof out.preamble !== "boolean") bad.push(`${at}: preamble has to be true or false`);
+  },
+  read(reply, { settings }) {
+    const { body, items } = splitItems(reply, (settings?.parse               ) || "csv", !!settings?.preamble);
+    return { value: items, body };
+  },
+  handsOn: items => (items.length ? null : "no items to hand on"),
+  render: items => ({ items: items.join(", ") }),
+  terms: items => items,
+  modifiers: () => [],
+};
+
+/**
+ * The List configuration that reproduces the retired `tags` kind exactly:
+ * what a v2 pipeline's `tags` job is upgraded to (docs/design/generic-lab.md
+ * §2.2). [rules] is the rules document of the dataset the pipeline was graded
+ * against, whose tag rules become Drop items and reply rules join Reject;
+ * [mode] the job's case.
+ *
+ * Reject goes first because the not-a-list test counts whitespace-separated
+ * pieces as the model wrote them, before numbering is tidied away; the rest
+ * are the old single pass's checks, in its order, as separate passes -- each
+ * only ever looked at what the one before it kept, so the same items go, for
+ * the same reasons.
+ */
+export function legacyTagsOut(rules                 , mode = "LOWER")   
+                                                                    
+  {
+  const all = rules?.rules || [];
+  const asItem = ({ id, why, match, examples }                           )           =>
+    ({ id, ...(why != null ? { why } : {}), ...(match ? { match } : {}), ...(examples ? { examples } : {}) });
+  const replyRules = all.filter(r => r.scope === "reply").map((r)           => {
+    const { tags, ...rest } = (r.match || {})                           ;
+    const ex = (r.examples || {})                                                               ;
+    return {
+      id: r.id, ...(r.why != null ? { why: r.why } : {}),
+      match: { ...rest, ...(tags ? { items: tags } : {}) },
+      examples: { rejects: ex.discards || ex.drops || [], keeps: ex.keeps || [] },
+    };
+  });
+  return {
+    kind: "list", parse: "csv", preamble: true,
+    modifiers: [
+      { type: "reject", rules: [
+        { id: "empty reply", why: "Nothing came back.", match: { blank: true } },
+        { id: "no values", why: "Separators and nothing between them.", match: { itemCount: { max: 0 } } },
+        { id: "prose, not a list", why: "More than half the items are five or more pieces of text: an answer, not a list.",
+          match: { items: { tokens: { min: 5 } }, share: { above: 0.5 } } },
+        ...replyRules,
+      ] },
+      { type: "tidy" },
+      { type: "drop", rules: all.filter(r => r.scope === "tag").map(asItem) },
+      { type: "drop-echoes" },
+      { type: "dedupe" },
+      { type: "keep-first", n: 24 },
+      { type: "case", mode },
+    ],
+  };
+}
+
+export default function list(core                                                                                                                              )       {
+  core.registerKinds({
+    outputKinds: { list: LIST_KIND                    },
+    modifiers: LIST_MODIFIERS                                 ,
+  });
+}