@mastra/evals 0.13.5-alpha.0 → 0.13.6-alpha.0

package/CHANGELOG.md

@@ -1,5 +1,25 @@
 # @mastra/evals
 
+## 0.13.6-alpha.0
+
+### Patch Changes
+
+- Export scorer utils from @mastra/evals/scorers/utils ([#7701](https://github.com/mastra-ai/mastra/pull/7701))
+
+- Add an answer similarity scorer ([#7586](https://github.com/mastra-ai/mastra/pull/7586))
+
+- Updated dependencies [[`b4379f7`](https://github.com/mastra-ai/mastra/commit/b4379f703fd74474f253420e8c3a684f2c4b2f8e), [`dd9119b`](https://github.com/mastra-ai/mastra/commit/dd9119b175a8f389082f75c12750e51f96d65dca), [`d34aaa1`](https://github.com/mastra-ai/mastra/commit/d34aaa1da5d3c5f991740f59e2fe6d28d3e2dd91), [`ce1e580`](https://github.com/mastra-ai/mastra/commit/ce1e580f6391e94a0c6816a9c5db0a21566a262f), [`b2babfa`](https://github.com/mastra-ai/mastra/commit/b2babfa9e75b22f2759179e71d8473f6dc5421ed), [`d8c3ba5`](https://github.com/mastra-ai/mastra/commit/d8c3ba516f4173282d293f7e64769cfc8738d360), [`a566c4e`](https://github.com/mastra-ai/mastra/commit/a566c4e92d86c1671707c54359b1d33934f7cc13), [`af333aa`](https://github.com/mastra-ai/mastra/commit/af333aa30fe6d1b127024b03a64736c46eddeca2), [`3863c52`](https://github.com/mastra-ai/mastra/commit/3863c52d44b4e5779968b802d977e87adf939d8e), [`6424c7e`](https://github.com/mastra-ai/mastra/commit/6424c7ec38b6921d66212431db1e0958f441b2a7), [`db94750`](https://github.com/mastra-ai/mastra/commit/db94750a41fd29b43eb1f7ce8e97ba8b9978c91b), [`a66a371`](https://github.com/mastra-ai/mastra/commit/a66a3716b00553d7f01842be9deb34f720b10fab), [`69fc3cd`](https://github.com/mastra-ai/mastra/commit/69fc3cd0fd814901785bdcf49bf536ab1e7fd975)]:
+  - @mastra/core@0.16.3-alpha.0
+
+## 0.13.5
+
+### Patch Changes
+
+- dependencies updates: ([#7544](https://github.com/mastra-ai/mastra/pull/7544))
+  - Updated dependency [`fs-extra@^11.3.1` ↗︎](https://www.npmjs.com/package/fs-extra/v/11.3.1) (from `^11.3.0`, in `dependencies`)
+- Updated dependencies [[`47b6dc9`](https://github.com/mastra-ai/mastra/commit/47b6dc94f4976d4f3d3882e8f19eb365bbc5976c), [`827d876`](https://github.com/mastra-ai/mastra/commit/827d8766f36a900afcaf64a040f7ba76249009b3), [`0662d02`](https://github.com/mastra-ai/mastra/commit/0662d02ef16916e67531890639fcd72c69cfb6e2), [`565d65f`](https://github.com/mastra-ai/mastra/commit/565d65fc16314a99f081975ec92f2636dff0c86d), [`6189844`](https://github.com/mastra-ai/mastra/commit/61898448e65bda02bb814fb15801a89dc6476938), [`4da3d68`](https://github.com/mastra-ai/mastra/commit/4da3d68a778e5c4d5a17351ef223289fe2f45a45), [`fd9bbfe`](https://github.com/mastra-ai/mastra/commit/fd9bbfee22484f8493582325f53e8171bf8e682b), [`7eaf1d1`](https://github.com/mastra-ai/mastra/commit/7eaf1d1cec7e828d7a98efc2a748ac395bbdba3b), [`6f046b5`](https://github.com/mastra-ai/mastra/commit/6f046b5ccc5c8721302a9a61d5d16c12374cc8d7), [`d7a8f59`](https://github.com/mastra-ai/mastra/commit/d7a8f59154b0621aec4f41a6b2ea2b3882f03cb7), [`0b0bbb2`](https://github.com/mastra-ai/mastra/commit/0b0bbb24f4198ead69792e92b68a350f52b45cf3), [`d951f41`](https://github.com/mastra-ai/mastra/commit/d951f41771e4e5da8da4b9f870949f9509e38756), [`4dda259`](https://github.com/mastra-ai/mastra/commit/4dda2593b6343f9258671de5fb237aeba3ef6bb7), [`8049e2e`](https://github.com/mastra-ai/mastra/commit/8049e2e8cce80a00353c64894c62b695ac34e35e), [`f3427cd`](https://github.com/mastra-ai/mastra/commit/f3427cdaf9eecd63360dfc897a4acbf5f4143a4e), [`defed1c`](https://github.com/mastra-ai/mastra/commit/defed1ca8040cc8d42e645c5a50a1bc52a4918d7), [`6991ced`](https://github.com/mastra-ai/mastra/commit/6991cedcb5a44a49d9fe58ef67926e1f96ba55b1), [`9cb9c42`](https://github.com/mastra-ai/mastra/commit/9cb9c422854ee81074989dd2d8dccc0500ba8d3e), [`8334859`](https://github.com/mastra-ai/mastra/commit/83348594d4f37b311ba4a94d679c5f8721d796d4), [`05f13b8`](https://github.com/mastra-ai/mastra/commit/05f13b8fb269ccfc4de98e9db58dbe16eae55a5e)]:
+  - @mastra/core@0.16.1
+
 ## 0.13.5-alpha.0
 
 ### Patch Changes

package/dist/{chunk-4LRZVFXR.js → chunk-KHEXN75Q.js}

@@ -1,7 +1,20 @@
+import { RuntimeContext } from '@mastra/core/runtime-context';
+
 // src/scorers/utils.ts
 var roundToTwoDecimals = (num) => {
   return Math.round((num + Number.EPSILON) * 100) / 100;
 };
+function isCloserTo(value, target1, target2) {
+  return Math.abs(value - target1) < Math.abs(value - target2);
+}
+var createTestRun = (input, output, additionalContext, runtimeContext) => {
+  return {
+    input: [{ role: "user", content: input }],
+    output: { role: "assistant", text: output },
+    additionalContext: additionalContext ?? {},
+    runtimeContext: runtimeContext ?? {}
+  };
+};
 var getUserMessageFromRunInput = (input) => {
   return input?.inputMessages.find(({ role }) => role === "user")?.content;
 };
@@ -37,6 +50,56 @@ var getCombinedSystemPrompt = (input) => {
 var getAssistantMessageFromRunOutput = (output) => {
   return output?.find(({ role }) => role === "assistant")?.content;
 };
+var createToolInvocation = ({
+  toolCallId,
+  toolName,
+  args,
+  result,
+  state = "result"
+}) => {
+  return {
+    toolCallId,
+    toolName,
+    args,
+    result,
+    state
+  };
+};
+var createUIMessage = ({
+  content,
+  role,
+  id = "test-message",
+  toolInvocations = []
+}) => {
+  return {
+    id,
+    role,
+    content,
+    parts: [{ type: "text", text: content }],
+    toolInvocations
+  };
+};
+var createAgentTestRun = ({
+  inputMessages = [],
+  output,
+  rememberedMessages = [],
+  systemMessages = [],
+  taggedSystemMessages = {},
+  runtimeContext = new RuntimeContext(),
+  runId = crypto.randomUUID()
+}) => {
+  return {
+    input: {
+      inputMessages,
+      rememberedMessages,
+      systemMessages,
+      taggedSystemMessages
+    },
+    output,
+    runtimeContext,
+    runId
+  };
+};
 function extractToolCalls(output) {
   const toolCalls = [];
   const toolCallInfos = [];
@@ -59,7 +122,13 @@ function extractToolCalls(output) {
   }
   return { tools: toolCalls, toolCallInfos };
 }
+var extractInputMessages = (runInput) => {
+  return runInput?.inputMessages?.map((msg) => msg.content) || [];
+};
+var extractAgentResponseMessages = (runOutput) => {
+  return runOutput.filter((msg) => msg.role === "assistant").map((msg) => msg.content);
+};
 
-export { extractToolCalls, getAssistantMessageFromRunOutput, getCombinedSystemPrompt, getUserMessageFromRunInput, roundToTwoDecimals };
-//# sourceMappingURL=chunk-4LRZVFXR.js.map
-//# sourceMappingURL=chunk-4LRZVFXR.js.map
+export { createAgentTestRun, createTestRun, createToolInvocation, createUIMessage, extractAgentResponseMessages, extractInputMessages, extractToolCalls, getAssistantMessageFromRunOutput, getCombinedSystemPrompt, getSystemMessagesFromRunInput, getUserMessageFromRunInput, isCloserTo, roundToTwoDecimals };
+//# sourceMappingURL=chunk-KHEXN75Q.js.map
+//# sourceMappingURL=chunk-KHEXN75Q.js.map

package/dist/chunk-KHEXN75Q.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/scorers/utils.ts"],"names":[],"mappings":";;;AAIO,IAAM,kBAAA,GAAqB,CAAC,GAAA,KAAgB;AACjD,EAAA,OAAO,KAAK,KAAA,CAAA,CAAO,GAAA,GAAM,MAAA,CAAO,OAAA,IAAW,GAAG,CAAA,GAAI,GAAA;AACpD;AAEO,SAAS,UAAA,CAAW,KAAA,EAAe,OAAA,EAAiB,OAAA,EAA0B;AACnF,EAAA,OAAO,IAAA,CAAK,IAAI,KAAA,GAAQ,OAAO,IAAI,IAAA,CAAK,GAAA,CAAI,QAAQ,OAAO,CAAA;AAC7D;AAeO,IAAM,aAAA,GAAgB,CAC3B,KAAA,EACA,MAAA,EACA,mBACA,cAAA,KACiB;AACjB,EAAA,OAAO;AAAA,IACL,OAAO,CAAC,EAAE,MAAM,MAAA,EAAQ,OAAA,EAAS,OAAO,CAAA;AAAA,IACxC,MAAA,EAAQ,EAAE,IAAA,EAAM,WAAA,EAAa,MAAM,MAAA,EAAO;AAAA,IAC1C,iBAAA,EAAmB,qBAAqB,EAAC;AAAA,IACzC,cAAA,EAAgB,kBAAkB;AAAC,GACrC;AACF;AAEO,IAAM,0BAAA,GAA6B,CAAC,KAAA,KAAmC;AAC5E,EAAA,OAAO,KAAA,EAAO,cAAc,IAAA,CAAK,CAAC,EAAE,IAAA,EAAK,KAAM,IAAA,KAAS,MAAM,CAAA,EAAG,OAAA;AACnE;AAEO,IAAM,6BAAA,GAAgC,CAAC,KAAA,KAA6C;AACzF,EAAA,MAAM,iBAA2B,EAAC;AAGlC,EAAA,IAAI,OAAO,cAAA,EAAgB;AACzB,IAAA,cAAA,CAAe,IAAA;AAAA,MACb,GAAG,KAAA,CAAM,cAAA,CACN,GAAA,CAAI,CAAA,GAAA,KAAO;AAEV,QAAA,IAAI,OAAO,GAAA,CAAI,OAAA,KAAY,QAAA,EAAU;AACnC,UAAA,OAAO,GAAA,CAAI,OAAA;AAAA,QACb,CAAA,MAAA,IAAW,KAAA,CAAM,OAAA,CAAQ,GAAA,CAAI,OAAO,CAAA,EAAG;AAErC,UAAA,OAAO,GAAA,CAAI,OAAA,CACR,MAAA,CAAO,CAAA,IAAA,KAAQ,KAAK,IAAA,KAAS,MAAM,CAAA,CACnC,GAAA,CAAI,UAAQ,IAAA,CAAK,IAAA,IAAQ,EAAE,CAAA,CAC3B,KAAK,GAAG,CAAA;AAAA,QACb;AACA,QAAA,OAAO,EAAA;AAAA,MACT,CAAC,CAAA,CACA,MAAA,CAAO,CAAA,OAAA,KAAW,OAAO;AAAA,KAC9B;AAAA,EACF;AAGA,EAAA,IAAI,OAAO,oBAAA,EAAsB;AAC/B,IAAA,MAAA,CAAO,MAAA,CAAO,KAAA,CAAM,oBAAoB,CAAA,CAAE,QAAQ,CAAA,QAAA,KAAY;AAC5D,MAAA,QAAA,CAAS,QAAQ,CAAA,GAAA,KAAO;AACtB,QAAA,IAAI,OAAO,GAAA,CAAI,OAAA,KAAY,QAAA,EAAU;AACnC,UAAA,cAAA,CAAe,IAAA,CAAK,IAAI,OAAO,CAAA;AAAA,QACjC;AAAA,MACF,CAAC,CAAA;AAAA,IACH,CAAC,CAAA;AAAA,EACH;AAEA,EAAA,OAAO,cAAA;AACT;AAEO,IAAM,uBAAA,GAA0B,CAAC,KAAA,KAA2C;AACjF,EAAA,MAAM,cAAA,GAAiB,8BAA8B,KAAK,CAAA;AAC1D,EAAA,OAAO,cAAA,CAAe,KAAK,MAAM,CAAA;AACnC;AAEO,IAAM,gCAAA,GAAmC,CAAC,MAAA,KAAqC;AACpF,EAAA,OAAO,MAAA,EAAQ,KAAK,CAAC,EAAE,MAAK,KAAM,IAAA,KAAS,WAAW,CAAA,EAAG,OAAA;AAC3D;AAEO,IAAM,uBAAuB,CAAC;AAAA,EACnC,UAAA;AAAA,EACA,QAAA;AAAA,EACA,IAAA;AAAA,EACA,MAAA;AAAA,EACA,KAAA,GAAQ;AACV,CAAA,KAMuH;AACrH,EAAA,OAAO;AAAA,IACL,UAAA;AAAA,IACA,QAAA;AAAA,IACA,IAAA;AAAA,IACA,MAAA;AAAA,IACA;AAAA,GACF;AACF;AAEO,IAAM,kBAAkB,CAAC;AAAA,EAC9B,OAAA;AAAA,EACA,IAAA;AAAA,EACA,EAAA,GAAK,cAAA;AAAA,EACL,kBAAkB;AACpB,CAAA,KAWiB;AACf,EAAA,OAAO;AAAA,IACL,EAAA;AAAA,IACA,IAAA;AAAA,IACA,OAAA;AAAA,IACA,OAAO,CAAC,EAAE,MAAM,MAAA,EAAQ,IAAA,EAAM,SAAS,CAAA;AAAA,IACvC;AAAA,GACF;AACF;AAEO,IAAM,qBAAqB,CAAC;AAAA,EACjC,gBAAgB,EAAC;AAAA,EACjB,MAAA;AAAA,EACA,qBAAqB,EAAC;AAAA,EACtB,iBAAiB,EAAC;AAAA,EAClB,uBAAuB,EAAC;AAAA,EACxB,cAAA,GAAiB,IAAI,cAAA,EAAe;AAAA,EACpC,KAAA,GAAQ,OAAO,UAAA;AACjB,CAAA,KAaK;AACH,EAAA,OAAO;AAAA,IACL,KAAA,EAAO;AAAA,MACL,aAAA;AAAA,MACA,kBAAA;AAAA,MACA,cAAA;AAAA,MACA;AAAA,KACF;AAAA,IACA,MAAA;AAAA,IACA,cAAA;AAAA,IACA;AAAA,GACF;AACF;AASO,SAAS,iBAAiB,MAAA,EAAqF;AACpH,EAAA,MAAM,YAAsB,EAAC;AAC7B,EAAA,MAAM,gBAAgC,EAAC;AAEvC,EAAA,KAAA,IAAS,YAAA,GAAe,CAAA,EAAG,YAAA,GAAe,MAAA,CAAO,QAAQ,YAAA,EAAA,EAAgB;AACvE,IAAA,MAAM,OAAA,GAAU,OAAO,YAAY,CAAA;AACnC,IAAA,IAAI,SAAS,eAAA,EAAiB;AAC5B,MAAA,KAAA,IAAS,kBAAkB,CAAA,EAAG,eAAA,GAAkB,OAAA,CAAQ,eAAA,CAAgB,QAAQ,eAAA,EAAA,EAAmB;AACjG,QAAA,MAAM,UAAA,GAAa,OAAA,CAAQ,eAAA,CAAgB,eAAe,CAAA;AAC1D,QAAA,IAAI,UAAA,IAAc,WAAW,QAAA,KAAa,UAAA,CAAW,UAAU,QAAA,IAAY,UAAA,CAAW,UAAU,MAAA,CAAA,EAAS;AACvG,UAAA,SAAA,CAAU,IAAA,CAAK,WAAW,QAAQ,CAAA;AAClC,UAAA,aAAA,CAAc,IAAA,CAAK;AAAA,YACjB,UAAU,UAAA,CAAW,QAAA;AAAA,YACrB,YAAY,UAAA,CAAW,UAAA,IAAc,CAAA,EAAG,YAAY,IAAI,eAAe,CAAA,CAAA;AAAA,YACvE,YAAA;AAAA,YACA;AAAA,WACD,CAAA;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,EAAA,OAAO,EAAE,KAAA,EAAO,SAAA,EAAW,aAAA,EAAc;AAC3C;AAEO,IAAM,oBAAA,GAAuB,CAAC,QAAA,KAA2D;AAC9F,EAAA,OAAO,UAAU,aAAA,EAAe,GAAA,CAAI,SAAO,GAAA,CAAI,OAAO,KAAK,EAAC;AAC9D;AAEO,IAAM,4BAAA,GAA+B,CAAC,SAAA,KAAiD;AAC5F,EAAA,OAAO,SAAA,CAAU,MAAA,CAAO,CAAA,GAAA,KAAO,GAAA,CAAI,IAAA,KAAS,WAAW,CAAA,CAAE,GAAA,CAAI,CAAA,GAAA,KAAO,GAAA,CAAI,OAAO,CAAA;AACjF","file":"chunk-KHEXN75Q.js","sourcesContent":["import { RuntimeContext } from '@mastra/core/runtime-context';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent, ScoringInput } from '@mastra/core/scores';\nimport type { ToolInvocation, UIMessage } from 'ai';\n\nexport const roundToTwoDecimals = (num: number) => {\n  return Math.round((num + Number.EPSILON) * 100) / 100;\n};\n\nexport function isCloserTo(value: number, target1: number, target2: number): boolean {\n  return Math.abs(value - target1) < Math.abs(value - target2);\n}\n\nexport type TestCase = {\n  input: string;\n  output: string;\n  expectedResult: {\n    score: number;\n    reason?: string;\n  };\n};\n\nexport type TestCaseWithContext = TestCase & {\n  context: string[];\n};\n\nexport const createTestRun = (\n  input: string,\n  output: string,\n  additionalContext?: Record<string, any>,\n  runtimeContext?: Record<string, any>,\n): ScoringInput => {\n  return {\n    input: [{ role: 'user', content: input }],\n    output: { role: 'assistant', text: output },\n    additionalContext: additionalContext ?? {},\n    runtimeContext: runtimeContext ?? {},\n  };\n};\n\nexport const getUserMessageFromRunInput = (input?: ScorerRunInputForAgent) => {\n  return input?.inputMessages.find(({ role }) => role === 'user')?.content;\n};\n\nexport const getSystemMessagesFromRunInput = (input?: ScorerRunInputForAgent): string[] => {\n  const systemMessages: string[] = [];\n\n  // Add standard system messages\n  if (input?.systemMessages) {\n    systemMessages.push(\n      ...input.systemMessages\n        .map(msg => {\n          // Handle different content types - extract text if it's an array of parts\n          if (typeof msg.content === 'string') {\n            return msg.content;\n          } else if (Array.isArray(msg.content)) {\n            // Extract text from parts array\n            return msg.content\n              .filter(part => part.type === 'text')\n              .map(part => part.text || '')\n              .join(' ');\n          }\n          return '';\n        })\n        .filter(content => content),\n    );\n  }\n\n  // Add tagged system messages (these are specialized system prompts)\n  if (input?.taggedSystemMessages) {\n    Object.values(input.taggedSystemMessages).forEach(messages => {\n      messages.forEach(msg => {\n        if (typeof msg.content === 'string') {\n          systemMessages.push(msg.content);\n        }\n      });\n    });\n  }\n\n  return systemMessages;\n};\n\nexport const getCombinedSystemPrompt = (input?: ScorerRunInputForAgent): string => {\n  const systemMessages = getSystemMessagesFromRunInput(input);\n  return systemMessages.join('\\n\\n');\n};\n\nexport const getAssistantMessageFromRunOutput = (output?: ScorerRunOutputForAgent) => {\n  return output?.find(({ role }) => role === 'assistant')?.content;\n};\n\nexport const createToolInvocation = ({\n  toolCallId,\n  toolName,\n  args,\n  result,\n  state = 'result',\n}: {\n  toolCallId: string;\n  toolName: string;\n  args: Record<string, any>;\n  result: Record<string, any>;\n  state?: ToolInvocation['state'];\n}): { toolCallId: string; toolName: string; args: Record<string, any>; result: Record<string, any>; state: string } => {\n  return {\n    toolCallId,\n    toolName,\n    args,\n    result,\n    state,\n  };\n};\n\nexport const createUIMessage = ({\n  content,\n  role,\n  id = 'test-message',\n  toolInvocations = [],\n}: {\n  id: string;\n  role: 'user' | 'assistant' | 'system';\n  content: string;\n  toolInvocations?: Array<{\n    toolCallId: string;\n    toolName: string;\n    args: Record<string, any>;\n    result: Record<string, any>;\n    state: any;\n  }>;\n}): UIMessage => {\n  return {\n    id,\n    role,\n    content,\n    parts: [{ type: 'text', text: content }],\n    toolInvocations,\n  };\n};\n\nexport const createAgentTestRun = ({\n  inputMessages = [],\n  output,\n  rememberedMessages = [],\n  systemMessages = [],\n  taggedSystemMessages = {},\n  runtimeContext = new RuntimeContext(),\n  runId = crypto.randomUUID(),\n}: {\n  inputMessages?: ScorerRunInputForAgent['inputMessages'];\n  output: ScorerRunOutputForAgent;\n  rememberedMessages?: ScorerRunInputForAgent['rememberedMessages'];\n  systemMessages?: ScorerRunInputForAgent['systemMessages'];\n  taggedSystemMessages?: ScorerRunInputForAgent['taggedSystemMessages'];\n  runtimeContext?: RuntimeContext;\n  runId?: string;\n}): {\n  input: ScorerRunInputForAgent;\n  output: ScorerRunOutputForAgent;\n  runtimeContext: RuntimeContext;\n  runId: string;\n} => {\n  return {\n    input: {\n      inputMessages,\n      rememberedMessages,\n      systemMessages,\n      taggedSystemMessages,\n    },\n    output,\n    runtimeContext,\n    runId,\n  };\n};\n\nexport type ToolCallInfo = {\n  toolName: string;\n  toolCallId: string;\n  messageIndex: number;\n  invocationIndex: number;\n};\n\nexport function extractToolCalls(output: ScorerRunOutputForAgent): { tools: string[]; toolCallInfos: ToolCallInfo[] } {\n  const toolCalls: string[] = [];\n  const toolCallInfos: ToolCallInfo[] = [];\n\n  for (let messageIndex = 0; messageIndex < output.length; messageIndex++) {\n    const message = output[messageIndex];\n    if (message?.toolInvocations) {\n      for (let invocationIndex = 0; invocationIndex < message.toolInvocations.length; invocationIndex++) {\n        const invocation = message.toolInvocations[invocationIndex];\n        if (invocation && invocation.toolName && (invocation.state === 'result' || invocation.state === 'call')) {\n          toolCalls.push(invocation.toolName);\n          toolCallInfos.push({\n            toolName: invocation.toolName,\n            toolCallId: invocation.toolCallId || `${messageIndex}-${invocationIndex}`,\n            messageIndex,\n            invocationIndex,\n          });\n        }\n      }\n    }\n  }\n\n  return { tools: toolCalls, toolCallInfos };\n}\n\nexport const extractInputMessages = (runInput: ScorerRunInputForAgent | undefined): string[] => {\n  return runInput?.inputMessages?.map(msg => msg.content) || [];\n};\n\nexport const extractAgentResponseMessages = (runOutput: ScorerRunOutputForAgent): string[] => {\n  return runOutput.filter(msg => msg.role === 'assistant').map(msg => msg.content);\n};\n"]}

package/dist/{chunk-EKSPLMYP.cjs → chunk-QKR2PMLZ.cjs}

@@ -1,9 +1,22 @@
 'use strict';
 
+var runtimeContext = require('@mastra/core/runtime-context');
+
 // src/scorers/utils.ts
 var roundToTwoDecimals = (num) => {
   return Math.round((num + Number.EPSILON) * 100) / 100;
 };
+function isCloserTo(value, target1, target2) {
+  return Math.abs(value - target1) < Math.abs(value - target2);
+}
+var createTestRun = (input, output, additionalContext, runtimeContext) => {
+  return {
+    input: [{ role: "user", content: input }],
+    output: { role: "assistant", text: output },
+    additionalContext: additionalContext ?? {},
+    runtimeContext: runtimeContext ?? {}
+  };
+};
 var getUserMessageFromRunInput = (input) => {
   return input?.inputMessages.find(({ role }) => role === "user")?.content;
 };
@@ -39,6 +52,56 @@ var getCombinedSystemPrompt = (input) => {
 var getAssistantMessageFromRunOutput = (output) => {
   return output?.find(({ role }) => role === "assistant")?.content;
 };
+var createToolInvocation = ({
+  toolCallId,
+  toolName,
+  args,
+  result,
+  state = "result"
+}) => {
+  return {
+    toolCallId,
+    toolName,
+    args,
+    result,
+    state
+  };
+};
+var createUIMessage = ({
+  content,
+  role,
+  id = "test-message",
+  toolInvocations = []
+}) => {
+  return {
+    id,
+    role,
+    content,
+    parts: [{ type: "text", text: content }],
+    toolInvocations
+  };
+};
+var createAgentTestRun = ({
+  inputMessages = [],
+  output,
+  rememberedMessages = [],
+  systemMessages = [],
+  taggedSystemMessages = {},
+  runtimeContext: runtimeContext$1 = new runtimeContext.RuntimeContext(),
+  runId = crypto.randomUUID()
+}) => {
+  return {
+    input: {
+      inputMessages,
+      rememberedMessages,
+      systemMessages,
+      taggedSystemMessages
+    },
+    output,
+    runtimeContext: runtimeContext$1,
+    runId
+  };
+};
 function extractToolCalls(output) {
   const toolCalls = [];
   const toolCallInfos = [];
@@ -61,11 +124,25 @@ function extractToolCalls(output) {
   }
   return { tools: toolCalls, toolCallInfos };
 }
+var extractInputMessages = (runInput) => {
+  return runInput?.inputMessages?.map((msg) => msg.content) || [];
+};
+var extractAgentResponseMessages = (runOutput) => {
+  return runOutput.filter((msg) => msg.role === "assistant").map((msg) => msg.content);
+};
 
+exports.createAgentTestRun = createAgentTestRun;
+exports.createTestRun = createTestRun;
+exports.createToolInvocation = createToolInvocation;
+exports.createUIMessage = createUIMessage;
+exports.extractAgentResponseMessages = extractAgentResponseMessages;
+exports.extractInputMessages = extractInputMessages;
 exports.extractToolCalls = extractToolCalls;
 exports.getAssistantMessageFromRunOutput = getAssistantMessageFromRunOutput;
 exports.getCombinedSystemPrompt = getCombinedSystemPrompt;
+exports.getSystemMessagesFromRunInput = getSystemMessagesFromRunInput;
 exports.getUserMessageFromRunInput = getUserMessageFromRunInput;
+exports.isCloserTo = isCloserTo;
 exports.roundToTwoDecimals = roundToTwoDecimals;
-//# sourceMappingURL=chunk-EKSPLMYP.cjs.map
-//# sourceMappingURL=chunk-EKSPLMYP.cjs.map
+//# sourceMappingURL=chunk-QKR2PMLZ.cjs.map
+//# sourceMappingURL=chunk-QKR2PMLZ.cjs.map

package/dist/chunk-QKR2PMLZ.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/scorers/utils.ts"],"names":["runtimeContext","RuntimeContext"],"mappings":";;;;;AAIO,IAAM,kBAAA,GAAqB,CAAC,GAAA,KAAgB;AACjD,EAAA,OAAO,KAAK,KAAA,CAAA,CAAO,GAAA,GAAM,MAAA,CAAO,OAAA,IAAW,GAAG,CAAA,GAAI,GAAA;AACpD;AAEO,SAAS,UAAA,CAAW,KAAA,EAAe,OAAA,EAAiB,OAAA,EAA0B;AACnF,EAAA,OAAO,IAAA,CAAK,IAAI,KAAA,GAAQ,OAAO,IAAI,IAAA,CAAK,GAAA,CAAI,QAAQ,OAAO,CAAA;AAC7D;AAeO,IAAM,aAAA,GAAgB,CAC3B,KAAA,EACA,MAAA,EACA,mBACA,cAAA,KACiB;AACjB,EAAA,OAAO;AAAA,IACL,OAAO,CAAC,EAAE,MAAM,MAAA,EAAQ,OAAA,EAAS,OAAO,CAAA;AAAA,IACxC,MAAA,EAAQ,EAAE,IAAA,EAAM,WAAA,EAAa,MAAM,MAAA,EAAO;AAAA,IAC1C,iBAAA,EAAmB,qBAAqB,EAAC;AAAA,IACzC,cAAA,EAAgB,kBAAkB;AAAC,GACrC;AACF;AAEO,IAAM,0BAAA,GAA6B,CAAC,KAAA,KAAmC;AAC5E,EAAA,OAAO,KAAA,EAAO,cAAc,IAAA,CAAK,CAAC,EAAE,IAAA,EAAK,KAAM,IAAA,KAAS,MAAM,CAAA,EAAG,OAAA;AACnE;AAEO,IAAM,6BAAA,GAAgC,CAAC,KAAA,KAA6C;AACzF,EAAA,MAAM,iBAA2B,EAAC;AAGlC,EAAA,IAAI,OAAO,cAAA,EAAgB;AACzB,IAAA,cAAA,CAAe,IAAA;AAAA,MACb,GAAG,KAAA,CAAM,cAAA,CACN,GAAA,CAAI,CAAA,GAAA,KAAO;AAEV,QAAA,IAAI,OAAO,GAAA,CAAI,OAAA,KAAY,QAAA,EAAU;AACnC,UAAA,OAAO,GAAA,CAAI,OAAA;AAAA,QACb,CAAA,MAAA,IAAW,KAAA,CAAM,OAAA,CAAQ,GAAA,CAAI,OAAO,CAAA,EAAG;AAErC,UAAA,OAAO,GAAA,CAAI,OAAA,CACR,MAAA,CAAO,CAAA,IAAA,KAAQ,KAAK,IAAA,KAAS,MAAM,CAAA,CACnC,GAAA,CAAI,UAAQ,IAAA,CAAK,IAAA,IAAQ,EAAE,CAAA,CAC3B,KAAK,GAAG,CAAA;AAAA,QACb;AACA,QAAA,OAAO,EAAA;AAAA,MACT,CAAC,CAAA,CACA,MAAA,CAAO,CAAA,OAAA,KAAW,OAAO;AAAA,KAC9B;AAAA,EACF;AAGA,EAAA,IAAI,OAAO,oBAAA,EAAsB;AAC/B,IAAA,MAAA,CAAO,MAAA,CAAO,KAAA,CAAM,oBAAoB,CAAA,CAAE,QAAQ,CAAA,QAAA,KAAY;AAC5D,MAAA,QAAA,CAAS,QAAQ,CAAA,GAAA,KAAO;AACtB,QAAA,IAAI,OAAO,GAAA,CAAI,OAAA,KAAY,QAAA,EAAU;AACnC,UAAA,cAAA,CAAe,IAAA,CAAK,IAAI,OAAO,CAAA;AAAA,QACjC;AAAA,MACF,CAAC,CAAA;AAAA,IACH,CAAC,CAAA;AAAA,EACH;AAEA,EAAA,OAAO,cAAA;AACT;AAEO,IAAM,uBAAA,GAA0B,CAAC,KAAA,KAA2C;AACjF,EAAA,MAAM,cAAA,GAAiB,8BAA8B,KAAK,CAAA;AAC1D,EAAA,OAAO,cAAA,CAAe,KAAK,MAAM,CAAA;AACnC;AAEO,IAAM,gCAAA,GAAmC,CAAC,MAAA,KAAqC;AACpF,EAAA,OAAO,MAAA,EAAQ,KAAK,CAAC,EAAE,MAAK,KAAM,IAAA,KAAS,WAAW,CAAA,EAAG,OAAA;AAC3D;AAEO,IAAM,uBAAuB,CAAC;AAAA,EACnC,UAAA;AAAA,EACA,QAAA;AAAA,EACA,IAAA;AAAA,EACA,MAAA;AAAA,EACA,KAAA,GAAQ;AACV,CAAA,KAMuH;AACrH,EAAA,OAAO;AAAA,IACL,UAAA;AAAA,IACA,QAAA;AAAA,IACA,IAAA;AAAA,IACA,MAAA;AAAA,IACA;AAAA,GACF;AACF;AAEO,IAAM,kBAAkB,CAAC;AAAA,EAC9B,OAAA;AAAA,EACA,IAAA;AAAA,EACA,EAAA,GAAK,cAAA;AAAA,EACL,kBAAkB;AACpB,CAAA,KAWiB;AACf,EAAA,OAAO;AAAA,IACL,EAAA;AAAA,IACA,IAAA;AAAA,IACA,OAAA;AAAA,IACA,OAAO,CAAC,EAAE,MAAM,MAAA,EAAQ,IAAA,EAAM,SAAS,CAAA;AAAA,IACvC;AAAA,GACF;AACF;AAEO,IAAM,qBAAqB,CAAC;AAAA,EACjC,gBAAgB,EAAC;AAAA,EACjB,MAAA;AAAA,EACA,qBAAqB,EAAC;AAAA,EACtB,iBAAiB,EAAC;AAAA,EAClB,uBAAuB,EAAC;AAAA,kBACxBA,gBAAA,GAAiB,IAAIC,6BAAA,EAAe;AAAA,EACpC,KAAA,GAAQ,OAAO,UAAA;AACjB,CAAA,KAaK;AACH,EAAA,OAAO;AAAA,IACL,KAAA,EAAO;AAAA,MACL,aAAA;AAAA,MACA,kBAAA;AAAA,MACA,cAAA;AAAA,MACA;AAAA,KACF;AAAA,IACA,MAAA;AAAA,oBACAD,gBAAA;AAAA,IACA;AAAA,GACF;AACF;AASO,SAAS,iBAAiB,MAAA,EAAqF;AACpH,EAAA,MAAM,YAAsB,EAAC;AAC7B,EAAA,MAAM,gBAAgC,EAAC;AAEvC,EAAA,KAAA,IAAS,YAAA,GAAe,CAAA,EAAG,YAAA,GAAe,MAAA,CAAO,QAAQ,YAAA,EAAA,EAAgB;AACvE,IAAA,MAAM,OAAA,GAAU,OAAO,YAAY,CAAA;AACnC,IAAA,IAAI,SAAS,eAAA,EAAiB;AAC5B,MAAA,KAAA,IAAS,kBAAkB,CAAA,EAAG,eAAA,GAAkB,OAAA,CAAQ,eAAA,CAAgB,QAAQ,eAAA,EAAA,EAAmB;AACjG,QAAA,MAAM,UAAA,GAAa,OAAA,CAAQ,eAAA,CAAgB,eAAe,CAAA;AAC1D,QAAA,IAAI,UAAA,IAAc,WAAW,QAAA,KAAa,UAAA,CAAW,UAAU,QAAA,IAAY,UAAA,CAAW,UAAU,MAAA,CAAA,EAAS;AACvG,UAAA,SAAA,CAAU,IAAA,CAAK,WAAW,QAAQ,CAAA;AAClC,UAAA,aAAA,CAAc,IAAA,CAAK;AAAA,YACjB,UAAU,UAAA,CAAW,QAAA;AAAA,YACrB,YAAY,UAAA,CAAW,UAAA,IAAc,CAAA,EAAG,YAAY,IAAI,eAAe,CAAA,CAAA;AAAA,YACvE,YAAA;AAAA,YACA;AAAA,WACD,CAAA;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,EAAA,OAAO,EAAE,KAAA,EAAO,SAAA,EAAW,aAAA,EAAc;AAC3C;AAEO,IAAM,oBAAA,GAAuB,CAAC,QAAA,KAA2D;AAC9F,EAAA,OAAO,UAAU,aAAA,EAAe,GAAA,CAAI,SAAO,GAAA,CAAI,OAAO,KAAK,EAAC;AAC9D;AAEO,IAAM,4BAAA,GAA+B,CAAC,SAAA,KAAiD;AAC5F,EAAA,OAAO,SAAA,CAAU,MAAA,CAAO,CAAA,GAAA,KAAO,GAAA,CAAI,IAAA,KAAS,WAAW,CAAA,CAAE,GAAA,CAAI,CAAA,GAAA,KAAO,GAAA,CAAI,OAAO,CAAA;AACjF","file":"chunk-QKR2PMLZ.cjs","sourcesContent":["import { RuntimeContext } from '@mastra/core/runtime-context';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent, ScoringInput } from '@mastra/core/scores';\nimport type { ToolInvocation, UIMessage } from 'ai';\n\nexport const roundToTwoDecimals = (num: number) => {\n  return Math.round((num + Number.EPSILON) * 100) / 100;\n};\n\nexport function isCloserTo(value: number, target1: number, target2: number): boolean {\n  return Math.abs(value - target1) < Math.abs(value - target2);\n}\n\nexport type TestCase = {\n  input: string;\n  output: string;\n  expectedResult: {\n    score: number;\n    reason?: string;\n  };\n};\n\nexport type TestCaseWithContext = TestCase & {\n  context: string[];\n};\n\nexport const createTestRun = (\n  input: string,\n  output: string,\n  additionalContext?: Record<string, any>,\n  runtimeContext?: Record<string, any>,\n): ScoringInput => {\n  return {\n    input: [{ role: 'user', content: input }],\n    output: { role: 'assistant', text: output },\n    additionalContext: additionalContext ?? {},\n    runtimeContext: runtimeContext ?? {},\n  };\n};\n\nexport const getUserMessageFromRunInput = (input?: ScorerRunInputForAgent) => {\n  return input?.inputMessages.find(({ role }) => role === 'user')?.content;\n};\n\nexport const getSystemMessagesFromRunInput = (input?: ScorerRunInputForAgent): string[] => {\n  const systemMessages: string[] = [];\n\n  // Add standard system messages\n  if (input?.systemMessages) {\n    systemMessages.push(\n      ...input.systemMessages\n        .map(msg => {\n          // Handle different content types - extract text if it's an array of parts\n          if (typeof msg.content === 'string') {\n            return msg.content;\n          } else if (Array.isArray(msg.content)) {\n            // Extract text from parts array\n            return msg.content\n              .filter(part => part.type === 'text')\n              .map(part => part.text || '')\n              .join(' ');\n          }\n          return '';\n        })\n        .filter(content => content),\n    );\n  }\n\n  // Add tagged system messages (these are specialized system prompts)\n  if (input?.taggedSystemMessages) {\n    Object.values(input.taggedSystemMessages).forEach(messages => {\n      messages.forEach(msg => {\n        if (typeof msg.content === 'string') {\n          systemMessages.push(msg.content);\n        }\n      });\n    });\n  }\n\n  return systemMessages;\n};\n\nexport const getCombinedSystemPrompt = (input?: ScorerRunInputForAgent): string => {\n  const systemMessages = getSystemMessagesFromRunInput(input);\n  return systemMessages.join('\\n\\n');\n};\n\nexport const getAssistantMessageFromRunOutput = (output?: ScorerRunOutputForAgent) => {\n  return output?.find(({ role }) => role === 'assistant')?.content;\n};\n\nexport const createToolInvocation = ({\n  toolCallId,\n  toolName,\n  args,\n  result,\n  state = 'result',\n}: {\n  toolCallId: string;\n  toolName: string;\n  args: Record<string, any>;\n  result: Record<string, any>;\n  state?: ToolInvocation['state'];\n}): { toolCallId: string; toolName: string; args: Record<string, any>; result: Record<string, any>; state: string } => {\n  return {\n    toolCallId,\n    toolName,\n    args,\n    result,\n    state,\n  };\n};\n\nexport const createUIMessage = ({\n  content,\n  role,\n  id = 'test-message',\n  toolInvocations = [],\n}: {\n  id: string;\n  role: 'user' | 'assistant' | 'system';\n  content: string;\n  toolInvocations?: Array<{\n    toolCallId: string;\n    toolName: string;\n    args: Record<string, any>;\n    result: Record<string, any>;\n    state: any;\n  }>;\n}): UIMessage => {\n  return {\n    id,\n    role,\n    content,\n    parts: [{ type: 'text', text: content }],\n    toolInvocations,\n  };\n};\n\nexport const createAgentTestRun = ({\n  inputMessages = [],\n  output,\n  rememberedMessages = [],\n  systemMessages = [],\n  taggedSystemMessages = {},\n  runtimeContext = new RuntimeContext(),\n  runId = crypto.randomUUID(),\n}: {\n  inputMessages?: ScorerRunInputForAgent['inputMessages'];\n  output: ScorerRunOutputForAgent;\n  rememberedMessages?: ScorerRunInputForAgent['rememberedMessages'];\n  systemMessages?: ScorerRunInputForAgent['systemMessages'];\n  taggedSystemMessages?: ScorerRunInputForAgent['taggedSystemMessages'];\n  runtimeContext?: RuntimeContext;\n  runId?: string;\n}): {\n  input: ScorerRunInputForAgent;\n  output: ScorerRunOutputForAgent;\n  runtimeContext: RuntimeContext;\n  runId: string;\n} => {\n  return {\n    input: {\n      inputMessages,\n      rememberedMessages,\n      systemMessages,\n      taggedSystemMessages,\n    },\n    output,\n    runtimeContext,\n    runId,\n  };\n};\n\nexport type ToolCallInfo = {\n  toolName: string;\n  toolCallId: string;\n  messageIndex: number;\n  invocationIndex: number;\n};\n\nexport function extractToolCalls(output: ScorerRunOutputForAgent): { tools: string[]; toolCallInfos: ToolCallInfo[] } {\n  const toolCalls: string[] = [];\n  const toolCallInfos: ToolCallInfo[] = [];\n\n  for (let messageIndex = 0; messageIndex < output.length; messageIndex++) {\n    const message = output[messageIndex];\n    if (message?.toolInvocations) {\n      for (let invocationIndex = 0; invocationIndex < message.toolInvocations.length; invocationIndex++) {\n        const invocation = message.toolInvocations[invocationIndex];\n        if (invocation && invocation.toolName && (invocation.state === 'result' || invocation.state === 'call')) {\n          toolCalls.push(invocation.toolName);\n          toolCallInfos.push({\n            toolName: invocation.toolName,\n            toolCallId: invocation.toolCallId || `${messageIndex}-${invocationIndex}`,\n            messageIndex,\n            invocationIndex,\n          });\n        }\n      }\n    }\n  }\n\n  return { tools: toolCalls, toolCallInfos };\n}\n\nexport const extractInputMessages = (runInput: ScorerRunInputForAgent | undefined): string[] => {\n  return runInput?.inputMessages?.map(msg => msg.content) || [];\n};\n\nexport const extractAgentResponseMessages = (runOutput: ScorerRunOutputForAgent): string[] => {\n  return runOutput.filter(msg => msg.role === 'assistant').map(msg => msg.content);\n};\n"]}

package/dist/scorers/code/index.cjs

@@ -1,6 +1,6 @@
 'use strict';
 
-var chunkEKSPLMYP_cjs = require('../../chunk-EKSPLMYP.cjs');
+var chunkQKR2PMLZ_cjs = require('../../chunk-QKR2PMLZ.cjs');
 var scores = require('@mastra/core/scores');
 var nlp = require('compromise');
 var difflib = require('difflib');
@@ -287,7 +287,7 @@ function createToolCallAccuracyScorerCode(options) {
     if (isInputInvalid || isOutputInvalid) {
       throw new Error("Input and output messages cannot be null or empty");
     }
-    const { tools: actualTools, toolCallInfos } = chunkEKSPLMYP_cjs.extractToolCalls(run.output);
+    const { tools: actualTools, toolCallInfos } = chunkQKR2PMLZ_cjs.extractToolCalls(run.output);
     const correctToolCalled = expectedTool ? strictMode ? actualTools.length === 1 && actualTools[0] === expectedTool : actualTools.includes(expectedTool) : false;
     return {
       expectedTool,

package/dist/scorers/code/index.js

@@ -1,4 +1,4 @@
-import { extractToolCalls } from '../../chunk-4LRZVFXR.js';
+import { extractToolCalls } from '../../chunk-KHEXN75Q.js';
 import { createScorer } from '@mastra/core/scores';
 import nlp from 'compromise';
 import { SequenceMatcher } from 'difflib';

package/dist/scorers/llm/answer-similarity/index.d.ts

@@ -0,0 +1,34 @@
+import type { MastraLanguageModel } from '@mastra/core/agent';
+import type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';
+export interface AnswerSimilarityOptions {
+    requireGroundTruth?: boolean;
+    semanticThreshold?: number;
+    exactMatchBonus?: number;
+    missingPenalty?: number;
+    contradictionPenalty?: number;
+    extraInfoPenalty?: number;
+    scale?: number;
+}
+export declare const ANSWER_SIMILARITY_DEFAULT_OPTIONS: Required<AnswerSimilarityOptions>;
+export declare const ANSWER_SIMILARITY_INSTRUCTIONS = "\nYou are a precise answer similarity evaluator for CI/CD testing. Your role is to compare agent outputs against ground truth answers to ensure consistency and accuracy in automated testing.\n\nKey Principles:\n1. Focus on semantic equivalence, not just string matching\n2. Recognize that different phrasings can convey the same information\n3. Identify missing critical information from the ground truth\n4. Detect contradictions between output and ground truth\n5. Provide actionable feedback for improving answer accuracy\n6. Be strict but fair - partial credit for partial matches\n";
+export declare function createAnswerSimilarityScorer({ model, options, }: {
+    model: MastraLanguageModel;
+    options?: AnswerSimilarityOptions;
+}): import("@mastra/core/scores").MastraScorer<string, ScorerRunInputForAgent, ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+    outputUnits: string[];
+    groundTruthUnits: string[];
+}> & Record<"analyzeStepResult", {
+    matches: {
+        groundTruthUnit: string;
+        outputUnit: string | null;
+        matchType: "exact" | "semantic" | "partial" | "missing";
+        explanation: string;
+    }[];
+    extraInOutput: string[];
+    contradictions: {
+        groundTruthUnit: string;
+        outputUnit: string;
+        explanation: string;
+    }[];
+}> & Record<"generateScoreStepResult", number> & Record<"generateReasonStepResult", string>>;
+//# sourceMappingURL=index.d.ts.map

package/dist/scorers/llm/answer-similarity/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/answer-similarity/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE9D,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAM3F,MAAM,WAAW,uBAAuB;IACtC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,eAAO,MAAM,iCAAiC,EAAE,QAAQ,CAAC,uBAAuB,CAQ/E,CAAC;AAEF,eAAO,MAAM,8BAA8B,+kBAU1C,CAAC;AA0BF,wBAAgB,4BAA4B,CAAC,EAC3C,KAAK,EACL,OAA2C,GAC5C,EAAE;IACD,KAAK,EAAE,mBAAmB,CAAC;IAC3B,OAAO,CAAC,EAAE,uBAAuB,CAAC;CACnC;;;;;;;;;;;;;;;;6FA4HA"}

package/dist/scorers/llm/answer-similarity/prompts.d.ts

@@ -0,0 +1,29 @@
+export declare const createExtractPrompt: ({ output, groundTruth }: {
+    output: string;
+    groundTruth: string;
+}) => string;
+export declare const createAnalyzePrompt: ({ outputUnits, groundTruthUnits, }: {
+    outputUnits: string[];
+    groundTruthUnits: string[];
+}) => string;
+export declare const createReasonPrompt: ({ output, groundTruth, score, analysis, scale, }: {
+    output: string;
+    groundTruth: string;
+    score: number;
+    analysis: {
+        matches: Array<{
+            groundTruthUnit: string;
+            outputUnit: string | null;
+            matchType: string;
+            explanation: string;
+        }>;
+        extraInOutput: string[];
+        contradictions: Array<{
+            outputUnit: string;
+            groundTruthUnit: string;
+            explanation: string;
+        }>;
+    };
+    scale: number;
+}) => string;
+//# sourceMappingURL=prompts.d.ts.map

package/dist/scorers/llm/answer-similarity/prompts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompts.d.ts","sourceRoot":"","sources":["../../../../src/scorers/llm/answer-similarity/prompts.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,mBAAmB,GAAI,yBAAyB;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,WAiCnG,CAAC;AAEF,eAAO,MAAM,mBAAmB,GAAI,oCAGjC;IACD,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B,WAgDA,CAAC;AAEF,eAAO,MAAM,kBAAkB,GAAI,kDAMhC;IACD,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE;QACR,OAAO,EAAE,KAAK,CAAC;YACb,eAAe,EAAE,MAAM,CAAC;YACxB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;YAC1B,SAAS,EAAE,MAAM,CAAC;YAClB,WAAW,EAAE,MAAM,CAAC;SACrB,CAAC,CAAC;QACH,aAAa,EAAE,MAAM,EAAE,CAAC;QACxB,cAAc,EAAE,KAAK,CAAC;YACpB,UAAU,EAAE,MAAM,CAAC;YACnB,eAAe,EAAE,MAAM,CAAC;YACxB,WAAW,EAAE,MAAM,CAAC;SACrB,CAAC,CAAC;KACJ,CAAC;IACF,KAAK,EAAE,MAAM,CAAC;CACf,WAuBA,CAAC"}