@mastra/evals 0.0.1-alpha.0

package/dist/evals.cjs.development.js

@@ -0,0 +1,1510 @@
+'use strict';
+
+Object.defineProperty(exports, '__esModule', { value: true });
+
+var core = require('@mastra/core');
+var zod = require('zod');
+var nlp = require('compromise');
+var stringSimilarity = require('string-similarity');
+var difflib = require('difflib');
+var keyword_extractor = require('keyword-extractor');
+var Sentiment = require('sentiment');
+
+function _arrayLikeToArray(r, a) {
+  (null == a || a > r.length) && (a = r.length);
+  for (var e = 0, n = Array(a); e < a; e++) n[e] = r[e];
+  return n;
+}
+function asyncGeneratorStep(n, t, e, r, o, a, c) {
+  try {
+    var i = n[a](c),
+      u = i.value;
+  } catch (n) {
+    return void e(n);
+  }
+  i.done ? t(u) : Promise.resolve(u).then(r, o);
+}
+function _asyncToGenerator(n) {
+  return function () {
+    var t = this,
+      e = arguments;
+    return new Promise(function (r, o) {
+      var a = n.apply(t, e);
+      function _next(n) {
+        asyncGeneratorStep(a, r, o, _next, _throw, "next", n);
+      }
+      function _throw(n) {
+        asyncGeneratorStep(a, r, o, _next, _throw, "throw", n);
+      }
+      _next(void 0);
+    });
+  };
+}
+function _createForOfIteratorHelperLoose(r, e) {
+  var t = "undefined" != typeof Symbol && r[Symbol.iterator] || r["@@iterator"];
+  if (t) return (t = t.call(r)).next.bind(t);
+  if (Array.isArray(r) || (t = _unsupportedIterableToArray(r)) || e && r && "number" == typeof r.length) {
+    t && (r = t);
+    var o = 0;
+    return function () {
+      return o >= r.length ? {
+        done: !0
+      } : {
+        done: !1,
+        value: r[o++]
+      };
+    };
+  }
+  throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
+}
+function _extends() {
+  return _extends = Object.assign ? Object.assign.bind() : function (n) {
+    for (var e = 1; e < arguments.length; e++) {
+      var t = arguments[e];
+      for (var r in t) ({}).hasOwnProperty.call(t, r) && (n[r] = t[r]);
+    }
+    return n;
+  }, _extends.apply(null, arguments);
+}
+function _inheritsLoose(t, o) {
+  t.prototype = Object.create(o.prototype), t.prototype.constructor = t, _setPrototypeOf(t, o);
+}
+function _regeneratorRuntime() {
+  _regeneratorRuntime = function () {
+    return e;
+  };
+  var t,
+    e = {},
+    r = Object.prototype,
+    n = r.hasOwnProperty,
+    o = Object.defineProperty || function (t, e, r) {
+      t[e] = r.value;
+    },
+    i = "function" == typeof Symbol ? Symbol : {},
+    a = i.iterator || "@@iterator",
+    c = i.asyncIterator || "@@asyncIterator",
+    u = i.toStringTag || "@@toStringTag";
+  function define(t, e, r) {
+    return Object.defineProperty(t, e, {
+      value: r,
+      enumerable: !0,
+      configurable: !0,
+      writable: !0
+    }), t[e];
+  }
+  try {
+    define({}, "");
+  } catch (t) {
+    define = function (t, e, r) {
+      return t[e] = r;
+    };
+  }
+  function wrap(t, e, r, n) {
+    var i = e && e.prototype instanceof Generator ? e : Generator,
+      a = Object.create(i.prototype),
+      c = new Context(n || []);
+    return o(a, "_invoke", {
+      value: makeInvokeMethod(t, r, c)
+    }), a;
+  }
+  function tryCatch(t, e, r) {
+    try {
+      return {
+        type: "normal",
+        arg: t.call(e, r)
+      };
+    } catch (t) {
+      return {
+        type: "throw",
+        arg: t
+      };
+    }
+  }
+  e.wrap = wrap;
+  var h = "suspendedStart",
+    l = "suspendedYield",
+    f = "executing",
+    s = "completed",
+    y = {};
+  function Generator() {}
+  function GeneratorFunction() {}
+  function GeneratorFunctionPrototype() {}
+  var p = {};
+  define(p, a, function () {
+    return this;
+  });
+  var d = Object.getPrototypeOf,
+    v = d && d(d(values([])));
+  v && v !== r && n.call(v, a) && (p = v);
+  var g = GeneratorFunctionPrototype.prototype = Generator.prototype = Object.create(p);
+  function defineIteratorMethods(t) {
+    ["next", "throw", "return"].forEach(function (e) {
+      define(t, e, function (t) {
+        return this._invoke(e, t);
+      });
+    });
+  }
+  function AsyncIterator(t, e) {
+    function invoke(r, o, i, a) {
+      var c = tryCatch(t[r], t, o);
+      if ("throw" !== c.type) {
+        var u = c.arg,
+          h = u.value;
+        return h && "object" == typeof h && n.call(h, "__await") ? e.resolve(h.__await).then(function (t) {
+          invoke("next", t, i, a);
+        }, function (t) {
+          invoke("throw", t, i, a);
+        }) : e.resolve(h).then(function (t) {
+          u.value = t, i(u);
+        }, function (t) {
+          return invoke("throw", t, i, a);
+        });
+      }
+      a(c.arg);
+    }
+    var r;
+    o(this, "_invoke", {
+      value: function (t, n) {
+        function callInvokeWithMethodAndArg() {
+          return new e(function (e, r) {
+            invoke(t, n, e, r);
+          });
+        }
+        return r = r ? r.then(callInvokeWithMethodAndArg, callInvokeWithMethodAndArg) : callInvokeWithMethodAndArg();
+      }
+    });
+  }
+  function makeInvokeMethod(e, r, n) {
+    var o = h;
+    return function (i, a) {
+      if (o === f) throw Error("Generator is already running");
+      if (o === s) {
+        if ("throw" === i) throw a;
+        return {
+          value: t,
+          done: !0
+        };
+      }
+      for (n.method = i, n.arg = a;;) {
+        var c = n.delegate;
+        if (c) {
+          var u = maybeInvokeDelegate(c, n);
+          if (u) {
+            if (u === y) continue;
+            return u;
+          }
+        }
+        if ("next" === n.method) n.sent = n._sent = n.arg;else if ("throw" === n.method) {
+          if (o === h) throw o = s, n.arg;
+          n.dispatchException(n.arg);
+        } else "return" === n.method && n.abrupt("return", n.arg);
+        o = f;
+        var p = tryCatch(e, r, n);
+        if ("normal" === p.type) {
+          if (o = n.done ? s : l, p.arg === y) continue;
+          return {
+            value: p.arg,
+            done: n.done
+          };
+        }
+        "throw" === p.type && (o = s, n.method = "throw", n.arg = p.arg);
+      }
+    };
+  }
+  function maybeInvokeDelegate(e, r) {
+    var n = r.method,
+      o = e.iterator[n];
+    if (o === t) return r.delegate = null, "throw" === n && e.iterator.return && (r.method = "return", r.arg = t, maybeInvokeDelegate(e, r), "throw" === r.method) || "return" !== n && (r.method = "throw", r.arg = new TypeError("The iterator does not provide a '" + n + "' method")), y;
+    var i = tryCatch(o, e.iterator, r.arg);
+    if ("throw" === i.type) return r.method = "throw", r.arg = i.arg, r.delegate = null, y;
+    var a = i.arg;
+    return a ? a.done ? (r[e.resultName] = a.value, r.next = e.nextLoc, "return" !== r.method && (r.method = "next", r.arg = t), r.delegate = null, y) : a : (r.method = "throw", r.arg = new TypeError("iterator result is not an object"), r.delegate = null, y);
+  }
+  function pushTryEntry(t) {
+    var e = {
+      tryLoc: t[0]
+    };
+    1 in t && (e.catchLoc = t[1]), 2 in t && (e.finallyLoc = t[2], e.afterLoc = t[3]), this.tryEntries.push(e);
+  }
+  function resetTryEntry(t) {
+    var e = t.completion || {};
+    e.type = "normal", delete e.arg, t.completion = e;
+  }
+  function Context(t) {
+    this.tryEntries = [{
+      tryLoc: "root"
+    }], t.forEach(pushTryEntry, this), this.reset(!0);
+  }
+  function values(e) {
+    if (e || "" === e) {
+      var r = e[a];
+      if (r) return r.call(e);
+      if ("function" == typeof e.next) return e;
+      if (!isNaN(e.length)) {
+        var o = -1,
+          i = function next() {
+            for (; ++o < e.length;) if (n.call(e, o)) return next.value = e[o], next.done = !1, next;
+            return next.value = t, next.done = !0, next;
+          };
+        return i.next = i;
+      }
+    }
+    throw new TypeError(typeof e + " is not iterable");
+  }
+  return GeneratorFunction.prototype = GeneratorFunctionPrototype, o(g, "constructor", {
+    value: GeneratorFunctionPrototype,
+    configurable: !0
+  }), o(GeneratorFunctionPrototype, "constructor", {
+    value: GeneratorFunction,
+    configurable: !0
+  }), GeneratorFunction.displayName = define(GeneratorFunctionPrototype, u, "GeneratorFunction"), e.isGeneratorFunction = function (t) {
+    var e = "function" == typeof t && t.constructor;
+    return !!e && (e === GeneratorFunction || "GeneratorFunction" === (e.displayName || e.name));
+  }, e.mark = function (t) {
+    return Object.setPrototypeOf ? Object.setPrototypeOf(t, GeneratorFunctionPrototype) : (t.__proto__ = GeneratorFunctionPrototype, define(t, u, "GeneratorFunction")), t.prototype = Object.create(g), t;
+  }, e.awrap = function (t) {
+    return {
+      __await: t
+    };
+  }, defineIteratorMethods(AsyncIterator.prototype), define(AsyncIterator.prototype, c, function () {
+    return this;
+  }), e.AsyncIterator = AsyncIterator, e.async = function (t, r, n, o, i) {
+    void 0 === i && (i = Promise);
+    var a = new AsyncIterator(wrap(t, r, n, o), i);
+    return e.isGeneratorFunction(r) ? a : a.next().then(function (t) {
+      return t.done ? t.value : a.next();
+    });
+  }, defineIteratorMethods(g), define(g, u, "Generator"), define(g, a, function () {
+    return this;
+  }), define(g, "toString", function () {
+    return "[object Generator]";
+  }), e.keys = function (t) {
+    var e = Object(t),
+      r = [];
+    for (var n in e) r.push(n);
+    return r.reverse(), function next() {
+      for (; r.length;) {
+        var t = r.pop();
+        if (t in e) return next.value = t, next.done = !1, next;
+      }
+      return next.done = !0, next;
+    };
+  }, e.values = values, Context.prototype = {
+    constructor: Context,
+    reset: function (e) {
+      if (this.prev = 0, this.next = 0, this.sent = this._sent = t, this.done = !1, this.delegate = null, this.method = "next", this.arg = t, this.tryEntries.forEach(resetTryEntry), !e) for (var r in this) "t" === r.charAt(0) && n.call(this, r) && !isNaN(+r.slice(1)) && (this[r] = t);
+    },
+    stop: function () {
+      this.done = !0;
+      var t = this.tryEntries[0].completion;
+      if ("throw" === t.type) throw t.arg;
+      return this.rval;
+    },
+    dispatchException: function (e) {
+      if (this.done) throw e;
+      var r = this;
+      function handle(n, o) {
+        return a.type = "throw", a.arg = e, r.next = n, o && (r.method = "next", r.arg = t), !!o;
+      }
+      for (var o = this.tryEntries.length - 1; o >= 0; --o) {
+        var i = this.tryEntries[o],
+          a = i.completion;
+        if ("root" === i.tryLoc) return handle("end");
+        if (i.tryLoc <= this.prev) {
+          var c = n.call(i, "catchLoc"),
+            u = n.call(i, "finallyLoc");
+          if (c && u) {
+            if (this.prev < i.catchLoc) return handle(i.catchLoc, !0);
+            if (this.prev < i.finallyLoc) return handle(i.finallyLoc);
+          } else if (c) {
+            if (this.prev < i.catchLoc) return handle(i.catchLoc, !0);
+          } else {
+            if (!u) throw Error("try statement without catch or finally");
+            if (this.prev < i.finallyLoc) return handle(i.finallyLoc);
+          }
+        }
+      }
+    },
+    abrupt: function (t, e) {
+      for (var r = this.tryEntries.length - 1; r >= 0; --r) {
+        var o = this.tryEntries[r];
+        if (o.tryLoc <= this.prev && n.call(o, "finallyLoc") && this.prev < o.finallyLoc) {
+          var i = o;
+          break;
+        }
+      }
+      i && ("break" === t || "continue" === t) && i.tryLoc <= e && e <= i.finallyLoc && (i = null);
+      var a = i ? i.completion : {};
+      return a.type = t, a.arg = e, i ? (this.method = "next", this.next = i.finallyLoc, y) : this.complete(a);
+    },
+    complete: function (t, e) {
+      if ("throw" === t.type) throw t.arg;
+      return "break" === t.type || "continue" === t.type ? this.next = t.arg : "return" === t.type ? (this.rval = this.arg = t.arg, this.method = "return", this.next = "end") : "normal" === t.type && e && (this.next = e), y;
+    },
+    finish: function (t) {
+      for (var e = this.tryEntries.length - 1; e >= 0; --e) {
+        var r = this.tryEntries[e];
+        if (r.finallyLoc === t) return this.complete(r.completion, r.afterLoc), resetTryEntry(r), y;
+      }
+    },
+    catch: function (t) {
+      for (var e = this.tryEntries.length - 1; e >= 0; --e) {
+        var r = this.tryEntries[e];
+        if (r.tryLoc === t) {
+          var n = r.completion;
+          if ("throw" === n.type) {
+            var o = n.arg;
+            resetTryEntry(r);
+          }
+          return o;
+        }
+      }
+      throw Error("illegal catch attempt");
+    },
+    delegateYield: function (e, r, n) {
+      return this.delegate = {
+        iterator: values(e),
+        resultName: r,
+        nextLoc: n
+      }, "next" === this.method && (this.arg = t), y;
+    }
+  }, e;
+}
+function _setPrototypeOf(t, e) {
+  return _setPrototypeOf = Object.setPrototypeOf ? Object.setPrototypeOf.bind() : function (t, e) {
+    return t.__proto__ = e, t;
+  }, _setPrototypeOf(t, e);
+}
+function _unsupportedIterableToArray(r, a) {
+  if (r) {
+    if ("string" == typeof r) return _arrayLikeToArray(r, a);
+    var t = {}.toString.call(r).slice(8, -1);
+    return "Object" === t && r.constructor && (t = r.constructor.name), "Map" === t || "Set" === t ? Array.from(r) : "Arguments" === t || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(t) ? _arrayLikeToArray(r, a) : void 0;
+  }
+}
+
+function evaluate(_x, _x2, _x3) {
+  return _evaluate.apply(this, arguments);
+}
+function _evaluate() {
+  _evaluate = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee(agent, input, metric) {
+    var agentOutput, metricResult;
+    return _regeneratorRuntime().wrap(function _callee$(_context) {
+      while (1) switch (_context.prev = _context.next) {
+        case 0:
+          _context.next = 2;
+          return agent.generate(input);
+        case 2:
+          agentOutput = _context.sent;
+          _context.next = 5;
+          return metric.measure({
+            input: input.toString(),
+            output: agentOutput.text
+          });
+        case 5:
+          metricResult = _context.sent;
+          // capture infomration about the evaluation
+          core.executeHook(core.AvailableHooks.ON_EVALUATION, {
+            input: input.toString(),
+            output: agentOutput.text,
+            result: metricResult
+          });
+          return _context.abrupt("return", metricResult);
+        case 8:
+        case "end":
+          return _context.stop();
+      }
+    }, _callee);
+  }));
+  return _evaluate.apply(this, arguments);
+}
+
+var MastraAgentJudge = function MastraAgentJudge(name, instructions, model) {
+  this.agent = void 0;
+  this.agent = new core.Agent({
+    name: "Mastra Eval Judge " + model.provider + " " + name,
+    instructions: instructions,
+    model: model
+  });
+};
+
+var ANSWER_RELEVANCY_AGENT_INSTRUCTIONS = "You are a balanced and nuanced answer relevancy evaluator. Your job is to determine if LLM outputs are relevant to the input, including handling partially relevant or uncertain cases.\n\nKey Principles:\n1. Evaluate whether the output addresses what the input is asking for\n2. Consider both direct answers and related context\n3. Prioritize relevance to the input over correctness\n4. Recognize that responses can be partially relevant\n5. Empty inputs or error messages should always be marked as \"no\"";
+function generateEvaluationStatementsPrompt(_ref) {
+  var output = _ref.output;
+  return "Given the text, break it down into meaningful statements while preserving context and relationships.\nDon't split too aggressively.\n\nSplit compound statements particularly when they:\n- Are joined by \"and\"\n- Contain multiple distinct facts or claims\n- Have multiple descriptive elements about the subject\n\n\nHandle special cases:\n- A single word answer should be treated as a complete statement\n- Error messages should be treated as a single statement\n- Empty strings should return an empty list\n- When splitting text, keep related information together\n\nExample:\nExample text: Look! A bird! Birds are an interesting animal.\n\n{{\n    \"statements\": [\"Look!\", \"A bird!\", \"Birds are interesting animals.\"]\n}}\n\nPlease return only JSON format with \"statements\" array.\nReturn empty list for empty input.\n\nText:\n" + output + "\n\nJSON:\n";
+}
+function generateEvaluatePrompt$3(_ref2) {
+  var input = _ref2.input,
+    statements = _ref2.statements;
+  return "Evaluate each statement's relevance to the input question, considering direct answers, related context, and uncertain cases.\n\n    Return JSON with array of verdict objects. Each verdict must include:\n    - \"verdict\": \"yes\", \"no\", or \"unsure\"\n    - \"reason\": Clear explanation of the verdict\n    - Exact match between number of verdicts and statements\n\n    Verdict Guidelines:\n    - \"yes\": Statement explicitly and directly answers the input question\n        * Contains specific answer to the question asked (e.g., \"The color of the sky is blue\")\n        * States explicit relationship between key concepts (e.g., \"X is the CEO of company Y\")\n        * Can stand alone as a complete answer\n        * Contains appropriate question-type response (e.g., location for \"where\", person for \"who\")\n\n    - \"unsure\": Statement shows partial relevance when it:\n        * Contains topic-related administrative/governance terms without direct answer\n        * Mentions locations or entities related to the answer without specifying their role\n        * References functions or characteristics typically associated with the answer\n        * Is incorrect but shows understanding of the question\n        * Uses importance indicators (\"main\", \"primary\", \"major\") with relevant concepts\n        * Includes indirect references to the answer (e.g., \"where the president works\")\n        * Contains multiple relevant concepts but lacks explicit relationship between them\n        * Demonstrates understanding of question domain without providing specific answer\n\n    - \"no\": Statement lacks meaningful connection to question when it:\n        * Contains no concepts related to the question type or domain\n        * Only mentions the broader topic without relevant details (e.g., \"the country has nice weather\")\n        * Provides general descriptions without addressing the specific question\n        * Contains purely tangential information about the subject\n        * Consists of empty or meaningless content\n        * Discusses characteristics unrelated to the question type (e.g., describing cuisine when asked about geography)\n        * Note: Assessment is about topical relationship, not factual accuracy\n\n    REMEMBER: A statmenent does not have to be correct, it just has to be relevant.\n    If the statement contains words or phrases that are relevant to the input, it is partially relevant.\n    If the statement is a direct answer to the input, it is relevant.\n    If the statement is completely unrelated to the input or contains nothing, it is not relevant.\n    DO NOT MAKE A JUDGEMENT ON THE CORRECTNESS OF THE STATEMENT, JUST THE RELEVANCY.\n\n\n    Example:\n    Input: \"What color is the sky during daytime?\"\n    Statements: [\n      \"The sky is blue during daytime\",\n      \"The sky is full of clouds\", \n      \"I had breakfast today\",\n      \"Blue is a beautiful color\",\n      \"Many birds fly in the sky\",\n      \"\",\n      \"The sky is purple during daytime\",\n    ]\n    JSON:\n    {{\n        \"verdicts\": [\n            {{\n                \"verdict\": \"yes\",\n                \"reason\": \"This statement explicitly answers what color the sky is during daytime\"\n            }},\n            {{\n                \"verdict\": \"unsure\",\n                \"reason\": \"This statement describes the sky but doesn't address its color\"\n            }},\n            {{\n                \"verdict\": \"no\",\n                \"reason\": \"This statement about breakfast is completely unrelated to the sky\"\n            }},\n            {{\n                \"verdict\": \"unsure\",\n                \"reason\": \"This statement about blue is related to color but doesn't address the sky\"\n            }},\n            {{\n                \"verdict\": \"unsure\",\n                \"reason\": \"This statement is about the sky but doesn't address its color\"\n            }},\n            {{\n                \"verdict\": \"no\",\n                \"reason\": \"This statement is empty\"\n            }},\n            {{\n                \"verdict\": \"unsure\",\n                \"reason\": \"This statement is incorrect but contains relevant information and still addresses the question\"\n            }}\n        ]\n    }}\n\n  Input:\n  " + input + "\n\n  Statements:\n  " + statements.join('\n') + "\n\n  JSON:\n  ";
+}
+function generateReasonPrompt$3(_ref3) {
+  var score = _ref3.score,
+    reasons = _ref3.reasons,
+    input = _ref3.input,
+    output = _ref3.output;
+  return "Explain the irrelevancy score (0-10) for the LLM's response using this context:\n  Context:\n  Input: " + input + "\n  Output: " + output + "\n  Score: " + score + "\n  Irrelevancy Reasons: " + reasons.join('\n') + "\n  \n  Rules:\n  - Explain score based on mix of direct answers and related context\n  - Consider both full and partial relevance\n  - Keep explanation concise and focused\n  - Use given score, don't recalculate\n  - Don't judge factual correctness\n  - Explain both relevant and irrelevant aspects\n  - For mixed responses, explain the balance\n\n    Format:\n    {\n        \"reason\": \"The score is {score} because {explanation of overall relevance}\"\n    }\n\n    Example Responses:\n    {\n        \"reason\": \"The score is 7 because while the first statement directly answers the question, the additional context is only partially relevant\"\n    }\n    {\n        \"reason\": \"The score is 3 because while the answer discusses the right topic, it doesn't directly address the question\"\n    }\n    ";
+}
+
+var AnswerRelevancyJudge = /*#__PURE__*/function (_MastraAgentJudge) {
+  function AnswerRelevancyJudge(model) {
+    return _MastraAgentJudge.call(this, 'Answer Relevancy', ANSWER_RELEVANCY_AGENT_INSTRUCTIONS, model) || this;
+  }
+  _inheritsLoose(AnswerRelevancyJudge, _MastraAgentJudge);
+  var _proto = AnswerRelevancyJudge.prototype;
+  _proto.evaluate = /*#__PURE__*/function () {
+    var _evaluate = /*#__PURE__*/_asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee(input, actualOutput) {
+      var statementPrompt, statements, prompt, result;
+      return _regeneratorRuntime().wrap(function _callee$(_context) {
+        while (1) switch (_context.prev = _context.next) {
+          case 0:
+            statementPrompt = generateEvaluationStatementsPrompt({
+              output: actualOutput
+            });
+            _context.next = 3;
+            return this.agent.generate(statementPrompt, {
+              output: zod.z.object({
+                statements: zod.z.array(zod.z.string())
+              })
+            });
+          case 3:
+            statements = _context.sent;
+            prompt = generateEvaluatePrompt$3({
+              input: input,
+              statements: statements.object.statements
+            });
+            _context.next = 7;
+            return this.agent.generate(prompt, {
+              output: zod.z.object({
+                verdicts: zod.z.array(zod.z.object({
+                  verdict: zod.z.string(),
+                  reason: zod.z.string()
+                }))
+              })
+            });
+          case 7:
+            result = _context.sent;
+            return _context.abrupt("return", result.object.verdicts);
+          case 9:
+          case "end":
+            return _context.stop();
+        }
+      }, _callee, this);
+    }));
+    function evaluate(_x, _x2) {
+      return _evaluate.apply(this, arguments);
+    }
+    return evaluate;
+  }();
+  _proto.getReason = /*#__PURE__*/function () {
+    var _getReason = /*#__PURE__*/_asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee2(input, actualOutput, score, reasons) {
+      var prompt, result;
+      return _regeneratorRuntime().wrap(function _callee2$(_context2) {
+        while (1) switch (_context2.prev = _context2.next) {
+          case 0:
+            prompt = generateReasonPrompt$3({
+              input: input,
+              output: actualOutput,
+              reasons: reasons,
+              score: score
+            });
+            _context2.next = 3;
+            return this.agent.generate(prompt, {
+              output: zod.z.object({
+                reason: zod.z.string()
+              })
+            });
+          case 3:
+            result = _context2.sent;
+            return _context2.abrupt("return", result.object.reason);
+          case 5:
+          case "end":
+            return _context2.stop();
+        }
+      }, _callee2, this);
+    }));
+    function getReason(_x3, _x4, _x5, _x6) {
+      return _getReason.apply(this, arguments);
+    }
+    return getReason;
+  }();
+  return AnswerRelevancyJudge;
+}(MastraAgentJudge);
+
+var AnswerRelevancyMetric = /*#__PURE__*/function (_Metric) {
+  function AnswerRelevancyMetric(model, _temp) {
+    var _this;
+    var _ref = _temp === void 0 ? {
+        uncertaintyWeight: 0.3,
+        scale: 10
+      } : _temp,
+      uncertaintyWeight = _ref.uncertaintyWeight,
+      scale = _ref.scale;
+    _this = _Metric.call(this) || this;
+    _this.judge = void 0;
+    _this.uncertaintyWeight = void 0;
+    _this.scale = void 0;
+    _this.uncertaintyWeight = uncertaintyWeight;
+    _this.judge = new AnswerRelevancyJudge(model);
+    _this.scale = scale;
+    return _this;
+  }
+  _inheritsLoose(AnswerRelevancyMetric, _Metric);
+  var _proto = AnswerRelevancyMetric.prototype;
+  _proto.measure = /*#__PURE__*/function () {
+    var _measure = /*#__PURE__*/_asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee(_ref2) {
+      var input, output, verdicts, score, reason;
+      return _regeneratorRuntime().wrap(function _callee$(_context) {
+        while (1) switch (_context.prev = _context.next) {
+          case 0:
+            input = _ref2.input, output = _ref2.output;
+            _context.next = 3;
+            return this.judge.evaluate(input, output);
+          case 3:
+            verdicts = _context.sent;
+            score = this.calculateScore(verdicts);
+            _context.next = 7;
+            return this.generateReason(input, output, score, verdicts);
+          case 7:
+            reason = _context.sent;
+            return _context.abrupt("return", {
+              score: score,
+              reason: reason
+            });
+          case 9:
+          case "end":
+            return _context.stop();
+        }
+      }, _callee, this);
+    }));
+    function measure(_x) {
+      return _measure.apply(this, arguments);
+    }
+    return measure;
+  }();
+  _proto.generateReason = /*#__PURE__*/function () {
+    var _generateReason = /*#__PURE__*/_asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee2(input, output, score, verdicts) {
+      var reasonsForVerdicts, _iterator, _step, _step$value, verdict, _reason, reason;
+      return _regeneratorRuntime().wrap(function _callee2$(_context2) {
+        while (1) switch (_context2.prev = _context2.next) {
+          case 0:
+            reasonsForVerdicts = [];
+            for (_iterator = _createForOfIteratorHelperLoose(verdicts || []); !(_step = _iterator()).done;) {
+              _step$value = _step.value, verdict = _step$value.verdict, _reason = _step$value.reason;
+              if (verdict.trim().toLowerCase() === 'no') {
+                reasonsForVerdicts.push(_reason);
+              }
+            }
+            _context2.next = 4;
+            return this.judge.getReason(input, output, score, reasonsForVerdicts);
+          case 4:
+            reason = _context2.sent;
+            return _context2.abrupt("return", reason);
+          case 6:
+          case "end":
+            return _context2.stop();
+        }
+      }, _callee2, this);
+    }));
+    function generateReason(_x2, _x3, _x4, _x5) {
+      return _generateReason.apply(this, arguments);
+    }
+    return generateReason;
+  }();
+  _proto.calculateScore = function calculateScore(evaluation) {
+    var numberOfVerdicts = (evaluation == null ? void 0 : evaluation.length) || 0;
+    if (numberOfVerdicts === 0) {
+      return 1;
+    }
+    var relevancyCount = 0;
+    for (var _iterator2 = _createForOfIteratorHelperLoose(evaluation), _step2; !(_step2 = _iterator2()).done;) {
+      var verdict = _step2.value.verdict;
+      if (verdict.trim().toLowerCase() === 'yes') {
+        relevancyCount++;
+      } else if (verdict.trim().toLowerCase() === 'unsure') {
+        relevancyCount += this.uncertaintyWeight;
+      }
+    }
+    var score = relevancyCount / numberOfVerdicts;
+    return Math.round(score * this.scale);
+  };
+  return AnswerRelevancyMetric;
+}(core.Metric);
+
+var CompletenessMetric = /*#__PURE__*/function (_Metric) {
+  function CompletenessMetric() {
+    return _Metric.apply(this, arguments) || this;
+  }
+  _inheritsLoose(CompletenessMetric, _Metric);
+  var _proto = CompletenessMetric.prototype;
+  _proto.measure = /*#__PURE__*/function () {
+    var _measure = /*#__PURE__*/_asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee(_ref) {
+      var input, output, inputDoc, outputDoc, inputElements, outputElements, coverage;
+      return _regeneratorRuntime().wrap(function _callee$(_context) {
+        while (1) switch (_context.prev = _context.next) {
+          case 0:
+            input = _ref.input, output = _ref.output;
+            if (!(input === null || input === undefined || output === null || output === undefined)) {
+              _context.next = 3;
+              break;
+            }
+            throw new Error('Inputs cannot be null or undefined');
+          case 3:
+            // Trim both inputs
+            input = input.trim();
+            output = output.trim();
+            inputDoc = nlp(input);
+            outputDoc = nlp(output); // Extract and log elements
+            inputElements = this.extractElements(inputDoc);
+            outputElements = this.extractElements(outputDoc); // Maybe we need a more sophisticated matching approach
+            coverage = this.calculateCoverage(inputElements, outputElements);
+            return _context.abrupt("return", {
+              score: coverage,
+              details: "Completeness score: " + (coverage * 100).toFixed(1) + "%",
+              confidence: 0.8,
+              metrics: {
+                inputElements: inputElements,
+                outputElements: outputElements,
+                missingElements: inputElements.filter(function (e) {
+                  return !outputElements.includes(e);
+                }),
+                elementCounts: {
+                  input: inputElements.length,
+                  output: outputElements.length
+                }
+              }
+            });
+          case 11:
+          case "end":
+            return _context.stop();
+        }
+      }, _callee, this);
+    }));
+    function measure(_x) {
+      return _measure.apply(this, arguments);
+    }
+    return measure;
+  }();
+  _proto.extractElements = function extractElements(doc) {
+    var _this = this;
+    // Get more specific elements and ensure they're arrays
+    var nouns = doc.nouns().out('array') || [];
+    var verbs = doc.verbs().toInfinitive().out('array') || [];
+    var topics = doc.topics().out('array') || [];
+    var terms = doc.terms().out('array') || [];
+    // Helper function to clean and split terms
+    var cleanAndSplitTerm = function cleanAndSplitTerm(term) {
+      // First normalize the string
+      var normalized = _this.normalizeString(term);
+      // Split on word boundaries and filter out empty strings
+      return normalized.replace(/([a-z])([A-Z])/g, '$1 $2') // Split camelCase
+      .replace(/[^a-z0-9]+/g, ' ') // Replace non-alphanumeric with spaces
+      .trim().split(/\s+/).filter(function (word) {
+        return word.length > 0;
+      });
+    };
+    // Process all elements
+    var processedTerms = [].concat(nouns.flatMap(cleanAndSplitTerm), verbs.flatMap(cleanAndSplitTerm), topics.flatMap(cleanAndSplitTerm), terms.flatMap(cleanAndSplitTerm));
+    // Remove duplicates
+    return [].concat(new Set(processedTerms));
+  };
+  _proto.normalizeString = function normalizeString(str) {
+    // Remove diacritics and convert to lowercase
+    return str.normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase();
+  };
+  _proto.calculateCoverage = function calculateCoverage(original, simplified) {
+    var _this2 = this;
+    if (original.length === 0) {
+      return simplified.length === 0 ? 1 : 0;
+    }
+    // Exact matching for short words (3 chars or less), substring matching for longer words
+    var covered = original.filter(function (element) {
+      return simplified.some(function (s) {
+        var elem = _this2.normalizeString(element);
+        var simp = _this2.normalizeString(s);
+        // For short words (3 chars or less), require exact match
+        if (elem.length <= 3) {
+          return elem === simp;
+        }
+        // For longer words, require substantial overlap (more than 60% of the longer word)
+        var longer = elem.length > simp.length ? elem : simp;
+        var shorter = elem.length > simp.length ? simp : elem;
+        if (longer.includes(shorter)) {
+          return shorter.length / longer.length > 0.6;
+        }
+        return false;
+      });
+    });
+    return covered.length / original.length;
+  };
+  return CompletenessMetric;
+}(core.Metric);
+
+var ContentSimilarityMetric = /*#__PURE__*/function (_Metric) {
+  function ContentSimilarityMetric(options) {
+    var _this;
+    if (options === void 0) {
+      options = {};
+    }
+    _this = _Metric.call(this) || this;
+    _this.options = void 0;
+    _this.options = _extends({
+      ignoreCase: true,
+      ignoreWhitespace: true
+    }, options);
+    return _this;
+  }
+  _inheritsLoose(ContentSimilarityMetric, _Metric);
+  var _proto = ContentSimilarityMetric.prototype;
+  _proto.measure = /*#__PURE__*/function () {
+    var _measure = /*#__PURE__*/_asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee(_ref) {
+      var input, output, processedInput, processedOutput, similarity;
+      return _regeneratorRuntime().wrap(function _callee$(_context) {
+        while (1) switch (_context.prev = _context.next) {
+          case 0:
+            input = _ref.input, output = _ref.output;
+            processedInput = input;
+            processedOutput = output;
+            if (this.options.ignoreCase) {
+              processedInput = processedInput.toLowerCase();
+              processedOutput = processedOutput.toLowerCase();
+            }
+            if (this.options.ignoreWhitespace) {
+              processedInput = processedInput.replace(/\s+/g, ' ').trim();
+              processedOutput = processedOutput.replace(/\s+/g, ' ').trim();
+            }
+            similarity = stringSimilarity.compareTwoStrings(processedInput, processedOutput);
+            return _context.abrupt("return", {
+              score: similarity,
+              details: "Content similarity: " + (similarity * 100).toFixed(1) + "%",
+              confidence: 0.9,
+              metrics: {
+                similarity: similarity
+              }
+            });
+          case 7:
+          case "end":
+            return _context.stop();
+        }
+      }, _callee, this);
+    }));
+    function measure(_x) {
+      return _measure.apply(this, arguments);
+    }
+    return measure;
+  }();
+  return ContentSimilarityMetric;
+}(core.Metric);
+
+var CONTEXT_POSITION_AGENT_INSTRUCTIONS = "You are a balanced and nuanced context position evaluator. Your job is to determine if retrieved context nodes are relevant to generating the expected output, with special attention to their ordering.\n\nKey Principles:\n1. Evaluate whether each context node contributes to understanding the expected output - both directly AND indirectly\n2. Consider all forms of relevance:\n   - Direct definitions or explanations\n   - Supporting evidence or examples\n   - Related characteristics or behaviors\n   - Real-world applications or effects\n3. Pay attention to the position of relevant information\n4. Recognize that earlier positions should contain more relevant information\n5. Be inclusive rather than exclusive in determining relevance - if the information supports or reinforces the output in any way, consider it relevant\n6. Empty or error nodes should be marked as not relevant";
+function generateEvaluatePrompt$2(_ref) {
+  var input = _ref.input,
+    output = _ref.output,
+    context = _ref.context;
+  return "Given the input, output, and context, evaluate each context piece's relevance by generating a list of JSON objects.\n\n**\nIMPORTANT: Your response must be in JSON format with a 'verdicts' key containing a list. Each verdict must have only two fields: `verdict` with either 'yes' or 'no', and `reason` explaining the verdict. Your reason should include relevant quotes from the context.\n\nExample Context: [\"The Sun is a star\", \"Stars produce their own light\", \"The Moon reflects sunlight\"]\nExample Query: \"What is the Sun?\"\nExample Expected Response: \"The Sun is a star that produces light.\"\n\nConsider context relevant if it:\n- Directly addresses the input\n- Provides examples or instances that help explain the concept\n- Offers related information that helps build understanding\n- Contains partial information that contributes to the response\n- Demonstrates or validates characteristics mentioned in the output\n- Shows real-world applications or effects of the concept\n- Reinforces or provides evidence for any part of the output\n- Helps establish credibility or understanding of the subject\n- Describes the actions the subject can perform\n\nA context piece should be considered relevant if it contributes ANY supporting information or evidence, even if indirect.\n\nExample:\n{\n    \"verdicts\": [\n        {\n            \"verdict\": \"yes\",\n            \"reason\": \"The context 'The Sun is a star' directly defines what the Sun is.\"\n        },\n        {\n            \"verdict\": \"yes\",\n            \"reason\": \"The context 'Stars produce their own light' is relevant as it describes a key characteristic of stars, which includes the Sun.\"\n        },\n        {\n            \"verdict\": \"no\",\n            \"reason\": \"The context 'The Moon reflects sunlight' is not relevant to defining what the Sun is.\"\n        }\n    ]  \n}\n\nThe number of verdicts MUST MATCH the number of context pieces exactly.\n**\n\nInput:\n" + input + "\n\nOutput:\n" + output + "\n\nContext:\n" + context + "\n\nJSON:\n";
+}
+function generateReasonPrompt$2(_ref2) {
+  var input = _ref2.input,
+    output = _ref2.output,
+    verdicts = _ref2.verdicts,
+    score = _ref2.score;
+  return "Given the input, output, verdicts, and position score, provide a BRIEF explanation for the score. Focus on both relevance and positioning of the context.\n  The retrieved contexts is a list containing `verdict` ('yes' or 'no' for relevance), `reason` (explaining the verdict) and `node` (the context text). Contexts are listed in their ranking order.\n\n**\nIMPORTANT: Return only JSON format with a single 'reason' key explaining the score.\nExample JSON:\n{\n    \"reason\": \"The score is <score> because <explanation>.\"\n}\n\nGuidelines:\n- Don't mention 'verdict' - refer to relevant/irrelevant nodes instead\n- Use information from the `reason` field, not the field itself\n- Reference node positions (first, second, etc.) when explaining relevance\n- For perfect scores (10.0), emphasize both relevance and optimal ordering\n- Always reference the ranking order when discussing relevance\n**\n\nPosition Score:\n" + score + "\n\nInput:\n" + input + "\n\nOutput:\n" + output + "\n\nContext:\n" + verdicts + "\n\nJSON:\n";
+}
+
+var ContextPositionJudge = /*#__PURE__*/function (_MastraAgentJudge) {
+  function ContextPositionJudge(model) {
+    return _MastraAgentJudge.call(this, 'Context Position', CONTEXT_POSITION_AGENT_INSTRUCTIONS, model) || this;
+  }
+  _inheritsLoose(ContextPositionJudge, _MastraAgentJudge);
+  var _proto = ContextPositionJudge.prototype;
+  _proto.evaluate = /*#__PURE__*/function () {
+    var _evaluate = /*#__PURE__*/_asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee(input, actualOutput, retrievalContext) {
+      var prompt, result;
+      return _regeneratorRuntime().wrap(function _callee$(_context) {
+        while (1) switch (_context.prev = _context.next) {
+          case 0:
+            prompt = generateEvaluatePrompt$2({
+              input: input,
+              output: actualOutput,
+              context: retrievalContext
+            });
+            _context.next = 3;
+            return this.agent.generate(prompt, {
+              output: zod.z.object({
+                verdicts: zod.z.array(zod.z.object({
+                  verdict: zod.z.string(),
+                  reason: zod.z.string()
+                }))
+              })
+            });
+          case 3:
+            result = _context.sent;
+            return _context.abrupt("return", result.object.verdicts);
+          case 5:
+          case "end":
+            return _context.stop();
+        }
+      }, _callee, this);
+    }));
+    function evaluate(_x, _x2, _x3) {
+      return _evaluate.apply(this, arguments);
+    }
+    return evaluate;
+  }();
+  _proto.getReason = /*#__PURE__*/function () {
+    var _getReason = /*#__PURE__*/_asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee2(input, actualOutput, score, verdicts) {
+      var prompt, result;
+      return _regeneratorRuntime().wrap(function _callee2$(_context2) {
+        while (1) switch (_context2.prev = _context2.next) {
+          case 0:
+            prompt = generateReasonPrompt$2({
+              input: input,
+              output: actualOutput,
+              verdicts: verdicts,
+              score: score
+            });
+            _context2.next = 3;
+            return this.agent.generate(prompt, {
+              output: zod.z.object({
+                reason: zod.z.string()
+              })
+            });
+          case 3:
+            result = _context2.sent;
+            return _context2.abrupt("return", result.object.reason);
+          case 5:
+          case "end":
+            return _context2.stop();
+        }
+      }, _callee2, this);
+    }));
+    function getReason(_x4, _x5, _x6, _x7) {
+      return _getReason.apply(this, arguments);
+    }
+    return getReason;
+  }();
+  return ContextPositionJudge;
+}(MastraAgentJudge);
+
+var ContextPositionMetric = /*#__PURE__*/function (_Metric) {
+  function ContextPositionMetric(model, _temp) {
+    var _this;
+    var _ref = _temp === void 0 ? {} : _temp,
+      _ref$scale = _ref.scale,
+      scale = _ref$scale === void 0 ? 10 : _ref$scale;
+    _this = _Metric.call(this) || this;
+    _this.judge = void 0;
+    _this.scale = void 0;
+    _this.judge = new ContextPositionJudge(model);
+    _this.scale = scale;
+    return _this;
+  }
+  _inheritsLoose(ContextPositionMetric, _Metric);
+  var _proto = ContextPositionMetric.prototype;
+  _proto.measure = /*#__PURE__*/function () {
+    var _measure = /*#__PURE__*/_asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee(_ref2) {
+      var input, output, context, verdicts, score, reason;
+      return _regeneratorRuntime().wrap(function _callee$(_context) {
+        while (1) switch (_context.prev = _context.next) {
+          case 0:
+            input = _ref2.input, output = _ref2.output, context = _ref2.context;
+            _context.next = 3;
+            return this.judge.evaluate(input, output, context);
+          case 3:
+            verdicts = _context.sent;
+            score = this.calculateScore(verdicts);
+            _context.next = 7;
+            return this.judge.getReason(input, output, score, verdicts);
+          case 7:
+            reason = _context.sent;
+            return _context.abrupt("return", {
+              score: score,
+              reason: reason
+            });
+          case 9:
+          case "end":
+            return _context.stop();
+        }
+      }, _callee, this);
+    }));
+    function measure(_x) {
+      return _measure.apply(this, arguments);
+    }
+    return measure;
+  }();
+  _proto.calculateScore = function calculateScore(verdicts) {
+    var totalVerdicts = (verdicts == null ? void 0 : verdicts.length) || 0;
+    if (totalVerdicts === 0) {
+      return 0;
+    }
+    // Convert to binary scores (1 for yes, 0 for no)
+    var binaryScores = verdicts.map(function (v) {
+      return v.verdict.trim().toLowerCase() === 'yes' ? 1 : 0;
+    });
+    var weightedSum = 0;
+    var maxPossibleSum = 0; // Track the maximum possible sum for normalization
+    // Calculate position-weighted scores
+    binaryScores.forEach(function (isRelevant, index) {
+      var positionWeight = 1 / (index + 1);
+      if (isRelevant) {
+        weightedSum += positionWeight;
+      }
+      maxPossibleSum += positionWeight; // Add to max possible sum regardless of relevance
+    });
+    if (weightedSum === 0) {
+      return 0;
+    }
+    // Normalize against the maximum possible score
+    var finalScore = weightedSum / maxPossibleSum * this.scale;
+    return finalScore;
+  };
+  return ContextPositionMetric;
+}(core.Metric);
+
+var CONTEXT_PRECISION_AGENT_INSTRUCTIONS = "You are a balanced and nuanced context precision evaluator. Your job is to determine if retrieved context nodes are relevant to generating the expected output.\n\nKey Principles:\n1. Evaluate whether each context node was useful in generating the expected output\n2. Consider both direct and indirect relevance\n3. Prioritize usefulness over completeness\n4. Recognize that some nodes may be partially relevant\n5. Empty or error nodes should be marked as not relevant";
+function generateEvaluatePrompt$1(_ref) {
+  var input = _ref.input,
+    output = _ref.output,
+    context = _ref.context;
+  return "Given the input, output, and context, evaluate each context piece's relevance by generating a list of JSON objects.\n\n**\nIMPORTANT: Your response must be in JSON format with a 'verdicts' key containing a list. Each verdict must have only two fields: `verdict` with either 'yes' or 'no', and `reason` explaining the verdict. Your reason should include relevant quotes from the context.\n\nExample Context: [\"The Sun is a star\", \"Stars produce their own light\", \"The Moon reflects sunlight\"]\nExample Query: \"What is the Sun?\"\nExample Expected Response: \"The Sun is a star that produces light.\"\n\nExample:\n{\n    \"verdicts\": [\n        {\n            \"verdict\": \"yes\",\n            \"reason\": \"The context 'The Sun is a star' directly defines what the Sun is.\"\n        },\n        {\n            \"verdict\": \"yes\",\n            \"reason\": \"The context 'Stars produce their own light' is relevant as it describes a key characteristic of stars, which includes the Sun.\"\n        },\n        {\n            \"verdict\": \"no\",\n            \"reason\": \"The context 'The Moon reflects sunlight' is not relevant to defining what the Sun is.\"\n        }\n    ]  \n}\n\nConsider context relevant if it:\n- Directly addresses the query\n- Provides examples or instances that help explain the concept\n- Offers related information that helps build understanding\n- Contains partial information that contributes to the response\n\nThe number of verdicts MUST MATCH the number of context pieces exactly.\n**\n\nInput:\n" + input + "\n\nOutput:\n" + output + "\n\nContext:\n" + context + "\n\nJSON:\n";
+}
+function generateReasonPrompt$1(_ref2) {
+  var input = _ref2.input,
+    output = _ref2.output,
+    verdicts = _ref2.verdicts,
+    score = _ref2.score;
+  return "Given the input, output, verdicts, and precision score, provide a BRIEF explanation for the score. Explain both its strengths and limitations.\nThe retrieved contexts is a list containing `verdict` ('yes' or 'no' for relevance), `reason` (explaining the verdict) and `node` (the context text). Contexts are listed in their ranking order.\n\n**\nIMPORTANT: Return only JSON format with a single 'reason' key explaining the score.\nExample JSON:\n{\n    \"reason\": \"The score is <score> because <explanation>.\"\n}\n\nGuidelines:\n- Don't mention 'verdict' - refer to relevant/irrelevant nodes instead\n- Use information from the `reason` field, not the field itself\n- Reference node positions (first, second, etc.) when explaining relevance\n- For perfect scores (10.0), emphasize both relevance and optimal ordering\n- Always reference the ranking order when discussing relevance\n**\n\nPrecision Score:\n" + score + "\n\nInput:\n" + input + "\n\nOutput:\n" + output + "\n\nContext:\n" + verdicts + "\n\nJSON:\n";
+}
+
+var ContextPrecisionJudge = /*#__PURE__*/function (_MastraAgentJudge) {
+  function ContextPrecisionJudge(model) {
+    return _MastraAgentJudge.call(this, 'Context Precision', CONTEXT_PRECISION_AGENT_INSTRUCTIONS, model) || this;
+  }
+  _inheritsLoose(ContextPrecisionJudge, _MastraAgentJudge);
+  var _proto = ContextPrecisionJudge.prototype;
+  _proto.evaluate = /*#__PURE__*/function () {
+    var _evaluate = /*#__PURE__*/_asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee(input, actualOutput, retrievalContext) {
+      var prompt, result;
+      return _regeneratorRuntime().wrap(function _callee$(_context) {
+        while (1) switch (_context.prev = _context.next) {
+          case 0:
+            prompt = generateEvaluatePrompt$1({
+              input: input,
+              output: actualOutput,
+              context: retrievalContext
+            });
+            _context.next = 3;
+            return this.agent.generate(prompt, {
+              output: zod.z.object({
+                verdicts: zod.z.array(zod.z.object({
+                  verdict: zod.z.string(),
+                  reason: zod.z.string()
+                }))
+              })
+            });
+          case 3:
+            result = _context.sent;
+            return _context.abrupt("return", result.object.verdicts);
+          case 5:
+          case "end":
+            return _context.stop();
+        }
+      }, _callee, this);
+    }));
+    function evaluate(_x, _x2, _x3) {
+      return _evaluate.apply(this, arguments);
+    }
+    return evaluate;
+  }();
+  _proto.getReason = /*#__PURE__*/function () {
+    var _getReason = /*#__PURE__*/_asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee2(input, actualOutput, score, verdicts) {
+      var prompt, result;
+      return _regeneratorRuntime().wrap(function _callee2$(_context2) {
+        while (1) switch (_context2.prev = _context2.next) {
+          case 0:
+            prompt = generateReasonPrompt$1({
+              input: input,
+              output: actualOutput,
+              verdicts: verdicts,
+              score: score
+            });
+            _context2.next = 3;
+            return this.agent.generate(prompt, {
+              output: zod.z.object({
+                reason: zod.z.string()
+              })
+            });
+          case 3:
+            result = _context2.sent;
+            return _context2.abrupt("return", result.object.reason);
+          case 5:
+          case "end":
+            return _context2.stop();
+        }
+      }, _callee2, this);
+    }));
+    function getReason(_x4, _x5, _x6, _x7) {
+      return _getReason.apply(this, arguments);
+    }
+    return getReason;
+  }();
+  return ContextPrecisionJudge;
+}(MastraAgentJudge);
+
+var ContextPrecisionMetric = /*#__PURE__*/function (_Metric) {
+  function ContextPrecisionMetric(model, _temp) {
+    var _this;
+    var _ref = _temp === void 0 ? {} : _temp,
+      _ref$scale = _ref.scale,
+      scale = _ref$scale === void 0 ? 10 : _ref$scale;
+    _this = _Metric.call(this) || this;
+    _this.judge = void 0;
+    _this.scale = void 0;
+    _this.judge = new ContextPrecisionJudge(model);
+    _this.scale = scale;
+    return _this;
+  }
+  _inheritsLoose(ContextPrecisionMetric, _Metric);
+  var _proto = ContextPrecisionMetric.prototype;
+  _proto.measure = /*#__PURE__*/function () {
+    var _measure = /*#__PURE__*/_asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee(_ref2) {
+      var input, output, context, verdicts, score, reason;
+      return _regeneratorRuntime().wrap(function _callee$(_context) {
+        while (1) switch (_context.prev = _context.next) {
+          case 0:
+            input = _ref2.input, output = _ref2.output, context = _ref2.context;
+            _context.next = 3;
+            return this.judge.evaluate(input, output, context);
+          case 3:
+            verdicts = _context.sent;
+            score = this.calculateScore(verdicts);
+            _context.next = 7;
+            return this.judge.getReason(input, output, score, verdicts);
+          case 7:
+            reason = _context.sent;
+            return _context.abrupt("return", {
+              score: score,
+              reason: reason
+            });
+          case 9:
+          case "end":
+            return _context.stop();
+        }
+      }, _callee, this);
+    }));
+    function measure(_x) {
+      return _measure.apply(this, arguments);
+    }
+    return measure;
+  }();
+  _proto.calculateScore = function calculateScore(verdicts) {
+    var totalVerdicts = (verdicts == null ? void 0 : verdicts.length) || 0;
+    if (totalVerdicts === 0) {
+      return 0;
+    }
+    // Convert to binary scores (1 for yes, 0 for no)
+    var binaryScores = verdicts.map(function (v) {
+      return v.verdict.trim().toLowerCase() === 'yes' ? 1 : 0;
+    });
+    var weightedPrecisionSum = 0;
+    var relevantCount = 0;
+    // Calculate weighted precision at each position
+    binaryScores.forEach(function (isRelevant, index) {
+      if (isRelevant) {
+        relevantCount++;
+        var currentPrecision = relevantCount / (index + 1);
+        weightedPrecisionSum += currentPrecision * isRelevant;
+      }
+    });
+    if (relevantCount === 0) {
+      return 0;
+    }
+    var finalScore = weightedPrecisionSum / relevantCount;
+    return finalScore * this.scale;
+  };
+  return ContextPrecisionMetric;
+}(core.Metric);
+
+var DifferenceMetric = /*#__PURE__*/function (_Metric) {
+  function DifferenceMetric() {
+    return _Metric.apply(this, arguments) || this;
+  }
+  _inheritsLoose(DifferenceMetric, _Metric);
+  var _proto = DifferenceMetric.prototype;
+  _proto.measure = /*#__PURE__*/function () {
+    var _measure = /*#__PURE__*/_asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee(_ref) {
+      var input, output, matcher, ratio, ops, changes, maxLength, lengthDiff, confidence;
+      return _regeneratorRuntime().wrap(function _callee$(_context) {
+        while (1) switch (_context.prev = _context.next) {
+          case 0:
+            input = _ref.input, output = _ref.output;
+            matcher = new difflib.SequenceMatcher(null, input, output);
+            ratio = matcher.ratio(); // Get detailed operations
+            ops = matcher.getOpcodes();
+            changes = ops.filter(function (_ref2) {
+              var op = _ref2[0];
+              return op !== 'equal';
+            }).length; // Calculate confidence based on text length difference
+            maxLength = Math.max(input.length, output.length);
+            lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;
+            confidence = 1 - lengthDiff;
+            return _context.abrupt("return", {
+              score: ratio,
+              details: "Difference score: " + (ratio * 100).toFixed(1) + "% with " + changes + " changes",
+              confidence: confidence,
+              metrics: {
+                ratio: ratio,
+                changes: changes,
+                lengthDiff: lengthDiff
+              }
+            });
+          case 9:
+          case "end":
+            return _context.stop();
+        }
+      }, _callee);
+    }));
+    function measure(_x) {
+      return _measure.apply(this, arguments);
+    }
+    return measure;
+  }();
+  return DifferenceMetric;
+}(core.Metric);
+
+var KeywordCoverageMetric = /*#__PURE__*/function (_Metric) {
+  function KeywordCoverageMetric() {
+    return _Metric.apply(this, arguments) || this;
+  }
+  _inheritsLoose(KeywordCoverageMetric, _Metric);
+  var _proto = KeywordCoverageMetric.prototype;
+  _proto.measure = /*#__PURE__*/function () {
+    var _measure = /*#__PURE__*/_asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee(_ref) {
+      var input, output, extractKeywords, referenceKeywords, responseKeywords, matchedKeywords, totalKeywords, coverage;
+      return _regeneratorRuntime().wrap(function _callee$(_context) {
+        while (1) switch (_context.prev = _context.next) {
+          case 0:
+            input = _ref.input, output = _ref.output;
+            if (!(!input && !output)) {
+              _context.next = 3;
+              break;
+            }
+            return _context.abrupt("return", {
+              score: 1,
+              details: 'Keyword coverage: 100.0% (0/0 keywords)',
+              confidence: 0.85,
+              metrics: {
+                totalKeywords: 0,
+                matchedKeywords: 0
+              }
+            });
+          case 3:
+            extractKeywords = function extractKeywords(text) {
+              return keyword_extractor.extract(text, {
+                language: 'english',
+                remove_digits: true,
+                return_changed_case: true,
+                remove_duplicates: true
+              });
+            };
+            referenceKeywords = new Set(extractKeywords(input));
+            responseKeywords = new Set(extractKeywords(output));
+            matchedKeywords = [].concat(referenceKeywords).filter(function (k) {
+              return responseKeywords.has(k);
+            });
+            totalKeywords = referenceKeywords.size;
+            coverage = totalKeywords > 0 ? matchedKeywords.length / totalKeywords : 0;
+            return _context.abrupt("return", {
+              score: coverage,
+              details: "Keyword coverage: " + (coverage * 100).toFixed(1) + "% (" + matchedKeywords.length + "/" + referenceKeywords.size + " keywords)",
+              confidence: 0.85,
+              metrics: {
+                totalKeywords: referenceKeywords.size,
+                matchedKeywords: matchedKeywords.length
+              }
+            });
+          case 10:
+          case "end":
+            return _context.stop();
+        }
+      }, _callee);
+    }));
+    function measure(_x) {
+      return _measure.apply(this, arguments);
+    }
+    return measure;
+  }();
+  return KeywordCoverageMetric;
+}(core.Metric);
+
+var PROMPT_ALIGNMENT_AGENT_INSTRUCTIONS = "You are a strict and thorough prompt alignment evaluator. Your job is to determine if LLM outputs follow their given prompt instructions exactly.\n\nKey Principles:\n1. Be EXTRA STRICT in your evaluation in regards to whether the instructions are followed exactly.\n2. Only give a \"yes\" verdict if an instruction is COMPLETELY followed\n3. Any partial compliance should be marked as \"no\"\n4. Provide clear, specific reasons for any \"no\" verdicts\n5. Focus solely on instruction compliance, not output quality\n\nRemember:\n- Each instruction must be evaluated independently\n- Verdicts must be either \"yes\" or \"no\" - no in-between\n- Reasons are required only for \"no\" verdicts\n- The number of verdicts must match the number of instructions exactly";
+function generateEvaluatePrompt(_ref) {
+  var instructions = _ref.instructions,
+    input = _ref.input,
+    output = _ref.output;
+  return "For the provided list of prompt instructions, determine whether each instruction has been followed in the LLM output.\nMake sure to judge the output on each instruction independently. Do not let instructions be influenced by other instructions.\nGenerate a list of verdicts in JSON format, where each verdict must have:\n- \"verdict\": Strictly \"yes\" or \"no\"\n- \"reason\": Give a reason for the verdict\n\nBe EXTRA STRICT in your evaluation. Only give \"yes\" if the instruction is followed COMPLETELY.\nEvaluate the output EXACTLY as written - consider every character, space, and case\n\nExample:\nInput: \"describe the sky\"\nOutput: \"the sky is Blue today\"\nInstructions: [\"Start sentences with capital letters\", \"Use proper English\"]\n\n{\n  \"verdicts\": [\n    {\n      \"verdict\": \"no\",\n      \"reason\": \"The sentence 'the sky is Blue' starts with lowercase 't'\"\n    },\n    {\n      \"verdict\": \"no\",\n      \"reason\": \"Improper capitalization: 'Blue' is capitalized mid-sentence\"\n    }\n  ]\n}\n\nPrompt Instructions:\n" + instructions.join('\n') + "\n\nInput:\n" + input + "\n\nLLM Actual Output:\n" + output + "\n\nJSON:";
+}
+function generateReasonPrompt(_ref2) {
+  var input = _ref2.input,
+    output = _ref2.output,
+    score = _ref2.score,
+    reasons = _ref2.reasons;
+  return "Explain the instruction following score (0-10) for the LLM's response using this context:\n  Context:\n  Input: " + input + "\n  Output: " + output + "\n  Score: " + score + "\n  Failure Reasons: " + reasons.join('\n') + "\n\n  Rules (follow these rules exactly. do not deviate):\n  - Keep your response concise and to the point.\n  - Do not change score from what is given.\n  - Do not make judgements on inputs or outputs (factual correctness, quality, etc).\n  - If there are failure reasons given, explain why the score is not higher.\n  \n\n  Output format:\n  {\n    \"reason\": \"The score is {score} because {explanation of instruction following}\"\n  }\n    \n  Example Responses:\n  {\n    \"reason\": \"The score is 10 because the output follows the instructions exactly\"\n  }\n  {\n    \"reason\": \"The score is 0 because the output does not follow the instructions\"\n  }\n  ";
+}
+
+var PromptAlignmentJudge = /*#__PURE__*/function (_MastraAgentJudge) {
+  function PromptAlignmentJudge(model) {
+    return _MastraAgentJudge.call(this, 'Prompt Alignment', PROMPT_ALIGNMENT_AGENT_INSTRUCTIONS, model) || this;
+  }
+  _inheritsLoose(PromptAlignmentJudge, _MastraAgentJudge);
+  var _proto = PromptAlignmentJudge.prototype;
+  _proto.evaluate = /*#__PURE__*/function () {
+    var _evaluate = /*#__PURE__*/_asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee(input, actualOutput, instructions) {
+      var prompt, result;
+      return _regeneratorRuntime().wrap(function _callee$(_context) {
+        while (1) switch (_context.prev = _context.next) {
+          case 0:
+            prompt = generateEvaluatePrompt({
+              input: input,
+              output: actualOutput,
+              instructions: instructions
+            });
+            _context.next = 3;
+            return this.agent.generate(prompt, {
+              output: zod.z.object({
+                verdicts: zod.z.array(zod.z.object({
+                  verdict: zod.z.string(),
+                  reason: zod.z.string()
+                }))
+              })
+            });
+          case 3:
+            result = _context.sent;
+            return _context.abrupt("return", result.object.verdicts);
+          case 5:
+          case "end":
+            return _context.stop();
+        }
+      }, _callee, this);
+    }));
+    function evaluate(_x, _x2, _x3) {
+      return _evaluate.apply(this, arguments);
+    }
+    return evaluate;
+  }();
+  _proto.getReason = /*#__PURE__*/function () {
+    var _getReason = /*#__PURE__*/_asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee2(input, actualOutput, score, reasons) {
+      var prompt, result;
+      return _regeneratorRuntime().wrap(function _callee2$(_context2) {
+        while (1) switch (_context2.prev = _context2.next) {
+          case 0:
+            prompt = generateReasonPrompt({
+              input: input,
+              output: actualOutput,
+              reasons: reasons,
+              score: score
+            });
+            _context2.next = 3;
+            return this.agent.generate(prompt, {
+              output: zod.z.object({
+                reason: zod.z.string()
+              })
+            });
+          case 3:
+            result = _context2.sent;
+            return _context2.abrupt("return", result.object.reason);
+          case 5:
+          case "end":
+            return _context2.stop();
+        }
+      }, _callee2, this);
+    }));
+    function getReason(_x4, _x5, _x6, _x7) {
+      return _getReason.apply(this, arguments);
+    }
+    return getReason;
+  }();
+  return PromptAlignmentJudge;
+}(MastraAgentJudge);
+
+var PromptAlignmentMetric = /*#__PURE__*/function (_Metric) {
+  function PromptAlignmentMetric(model, _ref) {
+    var _this;
+    var instructions = _ref.instructions,
+      _ref$scale = _ref.scale,
+      scale = _ref$scale === void 0 ? 10 : _ref$scale;
+    _this = _Metric.call(this) || this;
+    _this.instructions = void 0;
+    _this.judge = void 0;
+    _this.scale = void 0;
+    _this.instructions = instructions;
+    _this.judge = new PromptAlignmentJudge(model);
+    _this.scale = scale;
+    return _this;
+  }
+  _inheritsLoose(PromptAlignmentMetric, _Metric);
+  var _proto = PromptAlignmentMetric.prototype;
+  _proto.measure = /*#__PURE__*/function () {
+    var _measure = /*#__PURE__*/_asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee(_ref2) {
+      var input, output, verdicts, score, reason;
+      return _regeneratorRuntime().wrap(function _callee$(_context) {
+        while (1) switch (_context.prev = _context.next) {
+          case 0:
+            input = _ref2.input, output = _ref2.output;
+            _context.next = 3;
+            return this.judge.evaluate(input, output, this.instructions);
+          case 3:
+            verdicts = _context.sent;
+            score = this.calculateScore(verdicts);
+            _context.next = 7;
+            return this.generateReason(input, output, score, verdicts);
+          case 7:
+            reason = _context.sent;
+            return _context.abrupt("return", {
+              score: score,
+              reason: reason
+            });
+          case 9:
+          case "end":
+            return _context.stop();
+        }
+      }, _callee, this);
+    }));
+    function measure(_x) {
+      return _measure.apply(this, arguments);
+    }
+    return measure;
+  }();
+  _proto.generateReason = /*#__PURE__*/function () {
+    var _generateReason = /*#__PURE__*/_asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee2(input, output, score, verdicts) {
+      var reasonsForVerdicts, _iterator, _step, _step$value, verdict, _reason, reason;
+      return _regeneratorRuntime().wrap(function _callee2$(_context2) {
+        while (1) switch (_context2.prev = _context2.next) {
+          case 0:
+            reasonsForVerdicts = [];
+            for (_iterator = _createForOfIteratorHelperLoose(verdicts || []); !(_step = _iterator()).done;) {
+              _step$value = _step.value, verdict = _step$value.verdict, _reason = _step$value.reason;
+              if (verdict.trim().toLowerCase() === 'no') {
+                reasonsForVerdicts.push(_reason);
+              }
+            }
+            _context2.next = 4;
+            return this.judge.getReason(input, output, score, reasonsForVerdicts);
+          case 4:
+            reason = _context2.sent;
+            return _context2.abrupt("return", reason);
+          case 6:
+          case "end":
+            return _context2.stop();
+        }
+      }, _callee2, this);
+    }));
+    function generateReason(_x2, _x3, _x4, _x5) {
+      return _generateReason.apply(this, arguments);
+    }
+    return generateReason;
+  }();
+  _proto.calculateScore = function calculateScore(evaluation) {
+    var numberOfVerdicts = (evaluation == null ? void 0 : evaluation.length) || 0;
+    if (numberOfVerdicts === 0) {
+      return 1;
+    }
+    var alignmentCount = 0;
+    for (var _iterator2 = _createForOfIteratorHelperLoose(evaluation), _step2; !(_step2 = _iterator2()).done;) {
+      var verdict = _step2.value.verdict;
+      if (verdict.trim().toLowerCase() !== 'no') {
+        alignmentCount++;
+      }
+    }
+    var score = alignmentCount / numberOfVerdicts;
+    return score * this.scale;
+  };
+  return PromptAlignmentMetric;
+}(core.Metric);
+
+var ToneConsistencyMetric = /*#__PURE__*/function (_Metric) {
+  function ToneConsistencyMetric() {
+    var _this;
+    for (var _len = arguments.length, args = new Array(_len), _key = 0; _key < _len; _key++) {
+      args[_key] = arguments[_key];
+    }
+    _this = _Metric.call.apply(_Metric, [this].concat(args)) || this;
+    _this.sentiment = new Sentiment();
+    return _this;
+  }
+  _inheritsLoose(ToneConsistencyMetric, _Metric);
+  var _proto = ToneConsistencyMetric.prototype;
+  _proto.measure = /*#__PURE__*/function () {
+    var _measure = /*#__PURE__*/_asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee(_ref) {
+      var _this2 = this;
+      var input, output, responseSentiment, referenceSentiment, sentimentDiff, normalizedScore, sentences, sentiments, avgSentiment, variance, stability;
+      return _regeneratorRuntime().wrap(function _callee$(_context) {
+        while (1) switch (_context.prev = _context.next) {
+          case 0:
+            input = _ref.input, output = _ref.output;
+            responseSentiment = this.sentiment.analyze(input);
+            if (!output) {
+              _context.next = 7;
+              break;
+            }
+            // Compare sentiment with reference
+            referenceSentiment = this.sentiment.analyze(output);
+            sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);
+            normalizedScore = Math.max(0, 1 - sentimentDiff);
+            return _context.abrupt("return", {
+              score: normalizedScore,
+              details: "Tone consistency: " + (normalizedScore * 100).toFixed(1) + "%",
+              confidence: 0.75,
+              metrics: {
+                responseSentiment: responseSentiment.comparative,
+                referenceSentiment: referenceSentiment.comparative,
+                difference: sentimentDiff
+              }
+            });
+          case 7:
+            // Evaluate sentiment stability across response
+            sentences = input.match(/[^.!?]+[.!?]+/g) || [input];
+            sentiments = sentences.map(function (s) {
+              return _this2.sentiment.analyze(s).comparative;
+            });
+            avgSentiment = sentiments.reduce(function (a, b) {
+              return a + b;
+            }, 0) / sentiments.length;
+            variance = sentiments.reduce(function (sum, s) {
+              return sum + Math.pow(s - avgSentiment, 2);
+            }, 0) / sentiments.length;
+            stability = Math.max(0, 1 - variance);
+            return _context.abrupt("return", {
+              score: stability,
+              details: "Tone stability: " + (stability * 100).toFixed(1) + "%",
+              confidence: 0.7,
+              metrics: {
+                avgSentiment: avgSentiment,
+                sentimentVariance: variance
+              }
+            });
+          case 13:
+          case "end":
+            return _context.stop();
+        }
+      }, _callee, this);
+    }));
+    function measure(_x) {
+      return _measure.apply(this, arguments);
+    }
+    return measure;
+  }();
+  return ToneConsistencyMetric;
+}(core.Metric);
+
+exports.AnswerRelevancyMetric = AnswerRelevancyMetric;
+exports.CompletenessMetric = CompletenessMetric;
+exports.ContentSimilarityMetric = ContentSimilarityMetric;
+exports.ContextPositionMetric = ContextPositionMetric;
+exports.ContextPrecisionMetric = ContextPrecisionMetric;
+exports.DifferenceMetric = DifferenceMetric;
+exports.KeywordCoverageMetric = KeywordCoverageMetric;
+exports.PromptAlignmentMetric = PromptAlignmentMetric;
+exports.ToneConsistencyMetric = ToneConsistencyMetric;
+exports.evaluate = evaluate;
+//# sourceMappingURL=evals.cjs.development.js.map