evals-lab 0.1.2

package/lab/metrics/builtin.mjs

@@ -0,0 +1,353 @@
+// The lab's own metrics (docs/pipeline-model.md § Metrics): checks of a
+// reply, in the manner of promptfoo's assertions -- deterministic ones that
+// read the text, ones that read it against what production replied, and
+// model-graded ones that ask a grader. Each is one METRICS entry the core
+// registers, as it registers kinds/list.ts; a plugin adds more the same way.
+                                                                                                   
+
+const ok = (pass         , reason        , score = pass ? 1 : 0)                => ({ pass, score, reason });
+const str = (v         )         => (v == null ? "" : String(v));
+const num = (v         )                => (typeof v === "number" ? v : str(v).trim() && !Number.isNaN(Number(v)) ? Number(v) : null);
+/** A list option: one entry a line. */
+const lines = (v         )           => str(v).split("\n").map((l) => l.trim()).filter(Boolean);
+const short = (s        ) => (s.length > 60 ? `${s.slice(0, 57)}…` : s);
+const json = (s        )                                      => {
+  try { return { value: JSON.parse(s) }; } catch (e) { return { error: e instanceof Error ? e.message : String(e) }; }
+};
+/** JSON values equal as values: an object's keys in any order. */
+const sorted = (v         )          => Array.isArray(v) ? v.map(sorted)
+  : v && typeof v === "object" ? Object.fromEntries(Object.keys(v).sort().map((k) => [k, sorted((v                           )[k])])) : v;
+const same = (a         , b         ) => JSON.stringify(sorted(a)) === JSON.stringify(sorted(b));
+const needs = (m        , keys          , at        , bad          ) => {
+  for (const k of keys) if (!str(m[k]).trim()) bad.push(`${at} needs its ${k}`);
+};
+const validRegex = (m        , at        , bad          ) => {
+  try { new RegExp(str(m.pattern), str(m.flags)); } catch { bad.push(`${at}: ${str(m.pattern)} is not a regular expression`); }
+};
+
+/** A value at a path in [v]: `tip`, `items[0].name`. */
+export function atPath(v         , path        )          {
+  let at          = v;
+  for (const part of path.replace(/\[(\d+)\]/g, ".$1").split(".").filter(Boolean)) {
+    if (at == null || typeof at !== "object") return undefined;
+    at = (at                           )[part];
+  }
+  return at;
+}
+
+/** Where [v] breaks [schema] -- the parts of JSON Schema a Parse JSON uses --
+    or null where it holds. */
+export function schemaProblem(v         , schema         , at = "the reply")                {
+  if (!schema || typeof schema !== "object") return null;
+  const s = schema                           ;
+  const type = s.type;
+  const kind = v === null ? "null" : Array.isArray(v) ? "array" : typeof v === "number" && Number.isInteger(v) ? "integer" : typeof v;
+  const types = Array.isArray(type) ? type : type ? [type] : [];
+  if (types.length && !types.some((t) => t === kind || (t === "number" && kind === "integer"))) {
+    return `${at} is ${kind}, and the schema wants ${types.join(" or ")}`;
+  }
+  if (Array.isArray(s.enum) && !s.enum.some((e) => same(e, v))) return `${at} is ${JSON.stringify(v)}, not one of ${s.enum.map((e) => JSON.stringify(e)).join(", ")}`;
+  if (kind === "object") {
+    const o = v                           ;
+    for (const r of Array.isArray(s.required) ? s.required : []) if (!(String(r) in o)) return `${at} has no ${r}`;
+    const props = (s.properties ?? {})                           ;
+    for (const [k, sub] of Object.entries(props)) {
+      if (k in o) { const p = schemaProblem(o[k], sub, `${at}.${k}`); if (p) return p; }
+    }
+    if (s.additionalProperties === false) {
+      const extra = Object.keys(o).find((k) => !(k in props));
+      if (extra) return `${at} has ${extra}, which the schema does not allow`;
+    }
+  }
+  if (kind === "array" && s.items) {
+    const a = v             ;
+    for (let i = 0; i < a.length; i++) { const p = schemaProblem(a[i], s.items, `${at}[${i}]`); if (p) return p; }
+  }
+  return null;
+}
+
+/** Levenshtein distance, for the near-enough metric. */
+function distance(a        , b        )         {
+  const d = Array.from({ length: b.length + 1 }, (_, j) => j);
+  for (let i = 1; i <= a.length; i++) {
+    let prev = d[0] ;
+    d[0] = i;
+    for (let j = 1; j <= b.length; j++) {
+      const was = d[j] ;
+      d[j] = Math.min(d[j]  + 1, d[j - 1]  + 1, prev + (a[i - 1] === b[j - 1] ? 0 : 1));
+      prev = was;
+    }
+  }
+  return d[b.length] ;
+}
+
+/** Production's reply, or the reason a metric that compares with it cannot. */
+const production = (input             )         => {
+  if (input.production == null) throw new Error("this item has no production reply to compare with");
+  return input.production;
+};
+
+// ---- model-graded -----------------------------------------------------------
+
+/** What a grader said, read from the first JSON object in its reply. */
+function verdictOf(said        )                                                                           {
+  const m = /\{[\s\S]*\}/.exec(said);
+  const read = m ? json(m[0]) : { error: "no JSON" };
+  if (read.error || !read.value || typeof read.value !== "object") throw new Error(`the grader's reply could not be read: ${short(said)}`);
+  return read.value                           ;
+}
+const ask = async (ctx           , prompt        )                  => {
+  if (!ctx.ask) throw new Error("a model-graded metric needs a grader, and this run has none");
+  return ctx.ask(prompt);
+};
+const gradedPass = (v                              , threshold        )                => {
+  const score = typeof v.score === "number" ? Math.max(0, Math.min(1, v.score)) : v.pass === true ? 1 : 0;
+  const pass = typeof v.pass === "boolean" ? v.pass && score >= threshold : score >= threshold;
+  return { pass, score, reason: str(v.reason) || (pass ? "the grader passed it" : "the grader failed it") };
+};
+const REPLY = 'Reply with one JSON object and nothing else: {"pass": true or false, "score": a number from 0 to 1, "reason": "one sentence"}.';
+
+const metrics                              = {
+  equals: {
+    label: "Equals",
+    description: "Passes when the reply is exactly the value; as JSON, the same value with keys in any order.",
+    // As JSON, a reply equals the value as a value -- keys in any order;
+    // otherwise, as the text, once trimmed.
+    options: [{ key: "value", label: "Value", type: "textarea" }, { key: "json", label: "As JSON", type: "checkbox" }],
+    defaults: () => ({ value: "", json: true }),
+    score: (input, m) => {
+      const want = str(m.value).trim(), got = input.text.trim();
+      const a = json(got), b = json(want);
+      const equal = m.json !== false && !a.error && !b.error ? same(a.value, b.value) : got === want;
+      return ok(equal, equal ? "equal" : `${short(got)} is not ${short(want)}`);
+    },
+  },
+  contains: {
+    label: "Contains",
+    description: "Passes when the reply contains the value.",
+    options: [{ key: "value", label: "Value", type: "text" }, { key: "ignoreCase", label: "Ignore case", type: "checkbox" }],
+    defaults: () => ({ value: "", ignoreCase: false }),
+    validate: (m, at, bad) => needs(m, ["value"], at, bad),
+    score: (input, m) => {
+      const fold = (s        ) => (m.ignoreCase ? s.toLowerCase() : s);
+      const hit = fold(input.text).includes(fold(str(m.value)));
+      return ok(hit, hit ? `has ${short(str(m.value))}` : `has no ${short(str(m.value))}`);
+    },
+  },
+  "contains-any": {
+    label: "Contains any",
+    description: "Passes when the reply contains at least one of the values, one a line.",
+    options: [{ key: "values", label: "Values", type: "textarea" }, { key: "ignoreCase", label: "Ignore case", type: "checkbox" }],
+    defaults: () => ({ values: "", ignoreCase: false }),
+    validate: (m, at, bad) => needs(m, ["values"], at, bad),
+    score: (input, m) => {
+      const fold = (s        ) => (m.ignoreCase ? s.toLowerCase() : s);
+      const hit = lines(m.values).find((v) => fold(input.text).includes(fold(v)));
+      return ok(!!hit, hit ? `has ${short(hit)}` : `has none of ${lines(m.values).join(", ")}`);
+    },
+  },
+  "contains-all": {
+    label: "Contains all",
+    description: "Passes when the reply contains every value, one a line.",
+    options: [{ key: "values", label: "Values", type: "textarea" }, { key: "ignoreCase", label: "Ignore case", type: "checkbox" }],
+    defaults: () => ({ values: "", ignoreCase: false }),
+    validate: (m, at, bad) => needs(m, ["values"], at, bad),
+    score: (input, m) => {
+      const fold = (s        ) => (m.ignoreCase ? s.toLowerCase() : s);
+      const all = lines(m.values);
+      const missing = all.filter((v) => !fold(input.text).includes(fold(v)));
+      return ok(!missing.length, missing.length ? `has no ${missing.join(", ")}` : "has them all",
+        all.length ? (all.length - missing.length) / all.length : 1);
+    },
+  },
+  regex: {
+    label: "Matches",
+    description: "Passes when the reply matches the regular expression.",
+    options: [{ key: "pattern", label: "Pattern", type: "text" }, { key: "flags", label: "Flags", type: "text" }],
+    defaults: () => ({ pattern: "", flags: "" }),
+    validate: (m, at, bad) => { needs(m, ["pattern"], at, bad); validRegex(m, at, bad); },
+    score: (input, m) => {
+      const hit = new RegExp(str(m.pattern), str(m.flags)).test(input.text);
+      return ok(hit, hit ? `matches /${str(m.pattern)}/` : `does not match /${str(m.pattern)}/`);
+    },
+  },
+  "starts-with": {
+    label: "Starts with",
+    description: "Passes when the reply starts with the value.",
+    options: [{ key: "value", label: "Value", type: "text" }],
+    defaults: () => ({ value: "" }),
+    validate: (m, at, bad) => needs(m, ["value"], at, bad),
+    score: (input, m) => {
+      const hit = input.text.startsWith(str(m.value));
+      return ok(hit, hit ? `starts with ${short(str(m.value))}` : `starts ${JSON.stringify(input.text.slice(0, 12))}`);
+    },
+  },
+  "is-json": {
+    label: "Is JSON",
+    description: "Passes when the reply is JSON and, with a schema, holds to it.",
+    options: [{ key: "schema", label: "Schema", type: "textarea" }],
+    defaults: () => ({ schema: "" }),
+    validate: (m, at, bad) => { if (str(m.schema).trim() && json(str(m.schema)).error) bad.push(`${at}: the schema is not JSON`); },
+    score: (input, m) => {
+      const read = json(input.text);
+      if (read.error) return ok(false, `not JSON: ${read.error}`);
+      const schema = str(m.schema).trim() ? json(str(m.schema)).value : null;
+      const why = schemaProblem(read.value, schema);
+      return ok(!why, why ?? (schema ? "as the schema says" : "valid"));
+    },
+  },
+  "json-field": {
+    label: "JSON field",
+    description: "Reads one field of a JSON reply and compares it.",
+    options: [
+      { key: "path", label: "Field", type: "text" },
+      { key: "op", label: "Is", type: "select", choices: [
+        { value: "equals", label: "equal to" }, { value: "one-of", label: "one of" }, { value: "matches", label: "matching" }] },
+      { key: "value", label: "Value", type: "textarea" },
+    ],
+    defaults: () => ({ path: "", op: "equals", value: "" }),
+    validate: (m, at, bad) => { needs(m, ["path"], at, bad); if (m.op === "matches") validRegex({ ...m, pattern: m.value }, at, bad); },
+    score: (input, m) => {
+      const read = json(input.text);
+      if (read.error) return ok(false, `not JSON: ${read.error}`);
+      const v = atPath(read.value, str(m.path));
+      if (v === undefined) return ok(false, `has no ${str(m.path)}`);
+      const text = typeof v === "string" ? v : JSON.stringify(v);
+      const pass = m.op === "one-of" ? lines(m.value).includes(text)
+        : m.op === "matches" ? new RegExp(str(m.value)).test(text)
+        : text === str(m.value).trim();
+      return ok(pass, `${str(m.path)} is ${short(text)}`);
+    },
+  },
+  "item-count": {
+    label: "Item count",
+    description: "Passes when the number of items is within the bounds.",
+    options: [{ key: "min", label: "At least", type: "number" }, { key: "max", label: "At most", type: "number" }],
+    defaults: () => ({ min: null, max: null }),
+    score: (input, m) => {
+      const n = input.terms.length, min = num(m.min), max = num(m.max);
+      const pass = (min == null || n >= min) && (max == null || n <= max);
+      return ok(pass, `${n} items`);
+    },
+  },
+  "count-matches": {
+    label: "Count matches",
+    description: "Counts matches of the pattern: points, for the weighted mode.",
+    counts: true,
+    options: [{ key: "pattern", label: "Pattern", type: "text" }, { key: "flags", label: "Flags", type: "text" },
+              { key: "min", label: "At least", type: "number" }],
+    defaults: () => ({ pattern: "", flags: "i", min: null }),
+    validate: (m, at, bad) => { needs(m, ["pattern"], at, bad); validRegex(m, at, bad); },
+    score: (input, m) => {
+      const flags = str(m.flags).includes("g") ? str(m.flags) : `${str(m.flags)}g`;
+      const n = [...input.text.matchAll(new RegExp(str(m.pattern), flags))].length;
+      const min = num(m.min);
+      return { pass: min == null ? n > 0 : n >= min, score: n, reason: `${n} ${n === 1 ? "match" : "matches"}` };
+    },
+  },
+  latency: {
+    label: "Latency",
+    description: "Passes when the reply came back within the time set.",
+    options: [{ key: "max", label: "At most (ms)", type: "number" }],
+    defaults: () => ({ max: 5000 }),
+    validate: (m, at, bad) => { if (num(m.max) == null) bad.push(`${at} needs its most, in ms`); },
+    score: (input, m) => ok(input.ms <= (num(m.max) ?? 0), `${input.ms} ms`),
+  },
+  levenshtein: {
+    label: "Near",
+    description: "Passes when the reply is within a few edits of the value.",
+    options: [{ key: "value", label: "Value", type: "textarea" }, { key: "max", label: "Edits at most", type: "number" }],
+    defaults: () => ({ value: "", max: 5 }),
+    score: (input, m) => {
+      const want = str(m.value).trim(), got = input.text.trim();
+      const d = distance(got, want);
+      return ok(d <= (num(m.max) ?? 0), `${d} ${d === 1 ? "edit" : "edits"} away`,
+        1 - d / Math.max(1, Math.max(got.length, want.length)));
+    },
+  },
+  // ---- against production ----------------------------------------------------
+  "equals-production": {
+    label: "Same as production",
+    description: "Passes when the reply is what production replied to the same item.",
+    options: [],
+    defaults: () => ({}),
+    score: (input) => {
+      const p = production(input).trim(), got = input.text.trim();
+      const a = json(got), b = json(p);
+      const equal = !a.error && !b.error ? same(a.value, b.value) : got === p;
+      return ok(equal, equal ? "as production replied" : `production replied ${short(p)}`);
+    },
+  },
+  "fields-equal-production": {
+    label: "Fields as production",
+    description: "Passes when the fields listed are what production's reply had.",
+    options: [{ key: "fields", label: "Fields", type: "textarea" }],
+    defaults: () => ({ fields: "" }),
+    validate: (m, at, bad) => needs(m, ["fields"], at, bad),
+    score: (input, m) => {
+      const a = json(input.text), b = json(production(input));
+      if (a.error) return ok(false, `not JSON: ${a.error}`);
+      if (b.error) return ok(false, "production's reply was not JSON");
+      const fields = lines(m.fields);
+      const differ = fields.filter((f) => !same(atPath(a.value, f), atPath(b.value, f)));
+      return ok(!differ.length, differ.length ? `${differ.join(", ")} differ from production` : "the fields match production",
+        fields.length ? (fields.length - differ.length) / fields.length : 1);
+    },
+  },
+  "same-parse-outcome": {
+    label: "Parses as production",
+    description: "Passes when the reply parses as JSON exactly when production's did.",
+    options: [],
+    defaults: () => ({}),
+    score: (input) => {
+      const mine = !json(input.text).error, theirs = !json(production(input)).error;
+      return ok(mine === theirs, `${mine ? "parses" : "does not parse"}, production's ${theirs ? "parsed" : "did not"}`);
+    },
+  },
+  // ---- model-graded -------------------------------------------------------------
+  "llm-rubric": {
+    label: "Rubric",
+    description: "Asks the grader whether the reply meets the rubric.",
+    graded: true,
+    options: [{ key: "rubric", label: "Rubric", type: "textarea" }, { key: "threshold", label: "Pass at", type: "number" }],
+    defaults: () => ({ rubric: "", threshold: 0.5 }),
+    validate: (m, at, bad) => needs(m, ["rubric"], at, bad),
+    score: async (input, m, ctx) => gradedPass(verdictOf(await ask(ctx,
+      `You are grading an output against a rubric.\n\nRubric:\n${str(m.rubric)}\n\nOutput:\n${input.text}\n\n${REPLY}`)),
+      num(m.threshold) ?? 0.5),
+  },
+  factuality: {
+    label: "Factual",
+    description: "Asks the grader whether the reply agrees with the reference, or production's reply.",
+    graded: true,
+    options: [{ key: "reference", label: "Reference", type: "textarea" }, { key: "threshold", label: "Pass at", type: "number" }],
+    defaults: () => ({ reference: "", threshold: 0.5 }),
+    // A blank reference is production's reply.
+    score: async (input, m, ctx) => gradedPass(verdictOf(await ask(ctx,
+      `You are checking an output for factual consistency with a reference. Differences in wording or detail are `
+      + `fine; a contradiction, or a fact the reference does not support, is not.\n\nReference:\n`
+      + `${str(m.reference).trim() || production(input)}\n\nOutput:\n${input.text}\n\n${REPLY}`)), num(m.threshold) ?? 0.5),
+  },
+  "judge-vs-production": {
+    label: "Judged against production",
+    description: "Asks the grader whether the reply is better than, the same as or worse than production's.",
+    graded: true,
+    options: [{ key: "task", label: "Task", type: "textarea" }],
+    defaults: () => ({ task: "" }),
+    score: async (input, m, ctx) => {
+      const v = verdictOf(await ask(ctx,
+        `Two outputs answer the same task. Say whether the NEW one is better than, the same as, or worse than the `
+        + `PRODUCTION one.${str(m.task).trim() ? `\n\nTask:\n${str(m.task)}` : ""}\n\nPRODUCTION:\n${production(input)}\n\n`
+        + `NEW:\n${input.text}\n\nReply with one JSON object and nothing else: {"verdict": "better" or "same" or "worse", `
+        + `"reason": "one sentence"}.`));
+      const verdict = str(v.verdict).toLowerCase();
+      if (!["better", "same", "worse"].includes(verdict)) throw new Error(`the grader said ${short(JSON.stringify(v))}`);
+      return { pass: verdict !== "worse", score: verdict === "better" ? 1 : verdict === "same" ? 0.5 : 0,
+               reason: `${verdict}: ${str(v.reason)}` };
+    },
+  },
+};
+
+export default function register(core                                                                           )       {
+  core.registerKinds({ metrics });
+}

package/lab/presets.json

@@ -0,0 +1,62 @@
+[
+  {
+    "id": "repeat-1.2",
+    "label": "repeat 1.2",
+    "opts": {
+      "repeatPenalty": "1.2"
+    }
+  },
+  {
+    "id": "repeat-1.3",
+    "label": "repeat 1.3",
+    "opts": {
+      "repeatPenalty": "1.3"
+    }
+  },
+  {
+    "id": "frequency-0.5",
+    "label": "frequency 0.5",
+    "opts": {
+      "frequencyPenalty": "0.5"
+    }
+  },
+  {
+    "id": "presence-0.5",
+    "label": "presence 0.5",
+    "opts": {
+      "presencePenalty": "0.5"
+    }
+  },
+  {
+    "id": "dry",
+    "label": "DRY 0.8 / 1.75 / 2",
+    "opts": {
+      "dryMultiplier": "0.8",
+      "dryBase": "1.75",
+      "dryAllowedLength": "2"
+    }
+  },
+  {
+    "id": "dry-strong",
+    "label": "DRY 2 / 2 / 2",
+    "opts": {
+      "dryMultiplier": "2",
+      "dryBase": "2",
+      "dryAllowedLength": "2"
+    }
+  },
+  {
+    "id": "grammar",
+    "label": "grammar",
+    "opts": {
+      "grammar": "root ::= \" \"? item (\",\" \" \"? item){0,23}\nitem ::= word (\" \" word){0,3}\nword ::= [A-Za-z] [-A-Za-z']{0,23}"
+    }
+  },
+  {
+    "id": "grammar-numbered",
+    "label": "grammar + numbers",
+    "opts": {
+      "grammar": "root ::= \" \"? item (\",\" \" \"? item){0,23}\nitem ::= (num \". \")? word (\" \" word){0,3}\nnum ::= [0-9] [0-9]?\nword ::= [A-Za-z] [-A-Za-z']{0,23}"
+    }
+  }
+]