evals-lab 0.13.0 → 0.14.0

package/lab/kinds/list.mjs

@@ -304,10 +304,10 @@ export const FILTER_SEPARATORS = ["comma", "lines", "semicolon", "none"]
                 
                                                                            
                  
-                                                                           
-                                                                           
-                                                                       
-                
+                                                                          
+                                                                            
+                                                                            
+                                
                                              
                                                                             
                                                                     
@@ -342,32 +342,81 @@ function rewriteItem(op                                     )
   return t => t;
 }
 
-/** The function a Rewrite text filter's `op` names: Remove thinking, Remove
-    preamble, or Replace pattern (a regex, global, and its replacement). An
-    unknown op leaves the text as it is; a pattern that will not compile throws,
-    so Ignore failures can carry a run past it. */
-function rewriteText(op                                     )                           {
-  if (op?.kind === "thinking") return removeThinking;
-  if (op?.kind === "preamble") return removePreamble;
-  if (op?.kind === "replace") {
-    const re = new RegExp(String(op.pattern ?? ""), "g");
-    const to = typeof op.replacement === "string" ? op.replacement : "";
-    return text => text.replace(re, to);
+/** The function a Rewrite text filter's `op` names (#557): Remove thinking,
+    Remove preamble, Trim, Replace text (literal, every occurrence), Replace
+    pattern (a regex, global, and its replacement), Strip code fences, Change
+    case, Keep first (N lines or characters), or Extract (a pattern's first
+    match, its first group if it has one). It answers null where nothing is
+    left to keep -- Extract found no match -- which rejects the reply. An
+    unknown op leaves the text as it is; a pattern that will not compile
+    throws, so Ignore failures can carry a run past it. */
+function rewriteText(op                                     )                                  {
+  switch (op?.kind) {
+    case "thinking": return removeThinking;
+    case "preamble": return removePreamble;
+    case "trim": return text => text.trim();
+    case "literal": {
+      const find = String(op.find ?? "");
+      const to = typeof op.replacement === "string" ? op.replacement : "";
+      return text => (find ? text.split(find).join(to) : text);
+    }
+    case "replace": {
+      const re = new RegExp(String(op.pattern ?? ""), "g");
+      const to = typeof op.replacement === "string" ? op.replacement : "";
+      return text => text.replace(re, to);
+    }
+    // A fence's own lines go -- the opening one with its language, and the
+    // closing one -- wherever they sit, so a reply fenced whole or a fence
+    // after a line of preamble both read as what was inside.
+    case "fences": return text => text.replace(FENCE_LINE, "");
+    case "case": { const mode = op.mode          ; return text => applyCase(text, mode); }
+    case "first": {
+      const n = op.n          ;
+      return op.unit === "chars" ? text => text.slice(0, n) : text => text.split("\n").slice(0, n).join("\n");
+    }
+    case "extract": {
+      const re = new RegExp(String(op.pattern ?? ""));
+      return text => { const m = re.exec(text); return m ? (m.length > 1 ? m[1] ?? "" : m[0]) : null; };
+    }
+    default: return text => text;
   }
-  return text => text;
 }
 
+const FENCE_LINE = /^[ \t]*```[^`\n]*(?:\n|$)/gm;
+
+/** Keep first's units, for a Rewrite text: lines, or characters. */
+const TEXT_UNITS = ["lines", "chars"]         ;
+
+/** Why a Rewrite text op's pattern does not compile, as a sentence, or none. */
+const patternProblems = (pattern         , at        )           => {
+  if (typeof pattern !== "string" || !pattern) return [`${at} needs a pattern`];
+  try { new RegExp(pattern); return []; } catch { return [`${at}'s pattern does not compile`]; }
+};
+
 /** The lab's base filter types (#405). Generic code asks this registry by
     `filter.type`; it never names an id. `rewrite-text`/`reject-text` read the
-    raw reply before it is read into items (the Pre-format stage); the rest
-    read the items after (the Post-format stage). */
+    reply's text: Pre-format, or Post-format after a kind that reads no items
+    (#557); the rest read the items, Post-format after a kind that does. */
 export const FILTER_TYPES                                  = {
   "rewrite-text": {
-    label: "Rewrite text", about: "Remove thinking, a preamble or a pattern from the reply's text.", scope: "text", conditions: [],
+    label: "Rewrite text", about: "Trim, replace, unfence, recase, shorten or extract from the reply's text.", scope: "text", conditions: [],
     opProblems: (f, at) => opKindProblems(f.op, at, {
-      thinking: [], preamble: [], replace: ["pattern", "replacement"],
+      thinking: [], preamble: [], trim: [], literal: ["find", "replacement"], replace: ["pattern", "replacement"],
+      fences: [], case: ["mode"], first: ["n", "unit"], extract: ["pattern"],
+    }, op => {
+      if (op.kind === "literal" && (typeof op.find !== "string" || !op.find)) return [`${at} needs text to find`];
+      if (op.kind === "extract") return patternProblems(op.pattern, at);
+      if (op.kind === "case" && !CASE_MODES.includes(op.mode          )) return [`${at}'s case needs a mode`];
+      if (op.kind === "first") {
+        return [...(Number.isInteger(op.n) && (op.n          ) >= 1 ? [] : [`${at} needs a whole number of 1 or more`]),
+                ...(TEXT_UNITS.includes(op.unit         ) ? [] : [`${at} keeps lines or characters`])];
+      }
+      return [];
     }),
-    apply: (filter, reply) => ({ body: rewriteText(filter.op)(reply.body), items: reply.items, dropped: [], reject: null }),
+    apply(filter, reply) {
+      const body = rewriteText(filter.op)(reply.body);
+      return { body: body ?? reply.body, items: reply.items, dropped: [], reject: body == null ? filter.id : null };
+    },
   },
   "reject-text": {
     label: "Reject text", about: "Discard the reply when it is blank or matches a pattern.", scope: "text", conditions: ["blank", "regex"], rejects: true,
@@ -522,6 +571,8 @@ export function tryFilter(filter        )
       return { test, ok: rejected === !test.expectedOutput.trim(), got: rejected ? "rejected" : (scope === "text" ? out.body : out.items.join(", ")) };
     }
     if (scope === "text") {
+      // A Rewrite text that rejects (Extract, matching nothing) leaves nothing.
+      if (out.reject != null) return { test, ok: !test.expectedOutput.trim(), got: "rejected" };
       const got = out.body.trim();
       return { test, ok: got === test.expectedOutput.trim(), got };
     }

package/lab/run-evals.js

@@ -724,7 +724,9 @@ async function ask(to, c, prompt, dataUrl, schema = null) {
     if (!r.ok) return said({ error: j.error?.message || j.error || `HTTP ${r.status}` });
     const { raw, finishReason } = core.connectionReply(c, j);
     const error = core.connectionReplyProblem(c, raw, finishReason);
-    return said(error ? { raw, error } : { raw });
+    // Why it stopped, and the limit sent where the limit stopped it (#597).
+    const finish = core.replyFinish(c, req.body, finishReason);
+    return said(error ? { raw, error, ...finish } : { raw, ...finish });
   } catch (e) {
     if (e?.name === "TimeoutError" || e?.name === "AbortError") {
       return said({ error: `the model gave no reply within ${seconds}s — timed out` });