@mastra/evals 0.12.1 → 0.12.2-alpha.0

package/dist/evaluation.d.ts

@@ -1,5 +1,5 @@
-import type { Metric } from '@mastra/core';
 import type { Agent } from '@mastra/core/agent';
+import type { Metric } from '@mastra/core/eval';
 export declare function evaluate<T extends Agent>(agent: T, input: Parameters<T['generate']>[0], metric: Metric): Promise<import("@mastra/core").EvaluationResult>;
 export declare const getCurrentTestInfo: () => Promise<{
     testName: any;

package/dist/evaluation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evaluation.d.ts","sourceRoot":"","sources":["../src/evaluation.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAIhD,wBAAsB,QAAQ,CAAC,CAAC,SAAS,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,oDAyB5G;AAED,eAAO,MAAM,kBAAkB;;;cAwB9B,CAAC"}
+{"version":3,"file":"evaluation.d.ts","sourceRoot":"","sources":["../src/evaluation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAIhD,wBAAsB,QAAQ,CAAC,CAAC,SAAS,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,oDAyB5G;AAED,eAAO,MAAM,kBAAkB;;;cAwB9B,CAAC"}

package/dist/index.cjs

@@ -1,6 +1,6 @@
 'use strict';
 
-var core = require('@mastra/core');
+var _eval = require('@mastra/core/eval');
 var hooks = require('@mastra/core/hooks');
 var storage = require('@mastra/core/storage');
 var utils = require('@mastra/core/utils');
@@ -20,7 +20,7 @@ async function evaluate(agent, input, metric) {
     globalRunId = process.env[GLOBAL_RUN_ID_ENV_KEY] = crypto.randomUUID();
     console.warn('Global run id not set, you should run "globalSetup" from "@mastra/evals" before evaluating.');
   }
-  const metricResult = await core.evaluate({
+  const metricResult = await _eval.evaluate({
     agentName: agent.name,
     input,
     metric,
@@ -41,7 +41,7 @@ var getCurrentTestInfo = async () => {
     };
   }
   try {
-    const vitest = await import('./dist-BODKWAXM.cjs');
+    const vitest = await import('./dist-JQCAD3AD.cjs');
     if (typeof vitest !== "undefined" && vitest.expect?.getState) {
       const state = vitest.expect.getState();
       return {

package/dist/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/constants.ts","../src/evaluation.ts","../src/attachListeners.ts"],"names":["coreEvaluate","registerHook","AvailableHooks","storage","checkEvalStorageFields","TABLE_EVALS"],"mappings":";;;;;;;;AAAO,IAAM,qBAAA,GAAwB,wBAAA;;;ACMrC,eAAsB,QAAA,CAA0B,KAAA,EAAU,KAAA,EAAqC,MAAA,EAAgB;AAC7G,EAAA,MAAM,QAAA,GAAW,MAAM,kBAAA,EAAmB;AAC1C,EAAA,IAAI,WAAA,GAAc,OAAA,CAAQ,GAAA,CAAI,qBAAqB,CAAA;AACnD,EAAA,MAAM,KAAA,GAAQ,OAAO,UAAA,EAAW;AAChC,EAAA,MAAM,WAAA,GAAc,MAAM,KAAA,CAAM,QAAA,CAAS,KAAA,EAAO;AAAA,IAC9C;AAAA,GACD,CAAA;AAED,EAAA,IAAI,CAAC,WAAA,EAAa;AAChB,IAAA,WAAA,GAAc,OAAA,CAAQ,GAAA,CAAI,qBAAqB,CAAA,GAAI,OAAO,UAAA,EAAW;AACrE,IAAA,OAAA,CAAQ,KAAK,6FAA6F,CAAA;AAAA,EAC5G;AAEA,EAAA,MAAM,YAAA,GAAe,MAAMA,aAAA,CAAa;AAAA,IACtC,WAAW,KAAA,CAAM,IAAA;AAAA,IACjB,KAAA;AAAA,IACA,MAAA;AAAA,IACA,QAAQ,WAAA,CAAY,IAAA;AAAA,IACpB,WAAA;AAAA,IACA,KAAA;AAAA,IACA,QAAA;AAAA,IACA,cAAc,KAAA,CAAM;AAAA,GACrB,CAAA;AAED,EAAA,OAAO,YAAA;AACT;AAEO,IAAM,qBAAqB,YAAY;AAG5C,EAAA,IAAI,OAAO,MAAA,KAAW,WAAA,IAAe,MAAA,CAAO,QAAA,EAAU;AAEpD,IAAA,MAAM,KAAA,GAAQ,OAAO,QAAA,EAAS;AAC9B,IAAA,OAAO;AAAA,MACL,UAAU,KAAA,CAAM,eAAA;AAAA,MAChB,UAAU,KAAA,CAAM;AAAA,KAClB;AAAA,EACF;AAEA,EAAA,IAAI;AACF,IAAA,MAAM,MAAA,GAAS,MAAM,OAAO,qBAAQ,CAAA;AACpC,IAAA,IAAI,OAAO,MAAA,KAAW,WAAA,IAAe,MAAA,CAAO,QAAQ,QAAA,EAAU;AAC5D,MAAA,MAAM,KAAA,GAAQ,MAAA,CAAO,MAAA,CAAO,QAAA,EAAS;AACrC,MAAA,OAAO;AAAA,QACL,UAAU,KAAA,CAAM,eAAA;AAAA,QAChB,UAAU,KAAA,CAAM;AAAA,OAClB;AAAA,IACF;AAAA,EACF,CAAA,CAAA,MAAQ;AAAA,EAAC;AAET,EAAA,OAAO,MAAA;AACT,CAAA;AClDA,eAAsB,gBAAgB,MAAA,EAAiB;AACrD,EAAAC,kBAAA,CAAaC,oBAAA,CAAe,aAAA,EAAe,OAAM,WAAA,KAAe;AAC9D,IAAA,MAAMC,SAAA,GAAU,QAAQ,UAAA,EAAW;AACnC,IAAA,IAAIA,SAAA,EAAS;AAEX,MAAA,MAAM,MAAA,GAAS,QAAQ,SAAA,EAAU;AACjC,MAAA,MAAM,cAAA,GAAiBC,4BAAA,CAAuB,WAAA,EAAa,MAAM,CAAA;AACjE,MAAA,IAAI,CAAC,cAAA,EAAgB;AAErB,MAAA,MAAMD,UAAQ,MAAA,CAAO;AAAA,QACnB,SAAA,EAAWE,mBAAA;AAAA,QACX,MAAA,EAAQ;AAAA,UACN,OAAO,WAAA,CAAY,KAAA;AAAA,UACnB,QAAQ,WAAA,CAAY,MAAA;AAAA,UACpB,QAAQ,IAAA,CAAK,SAAA,CAAU,WAAA,CAAY,MAAA,IAAU,EAAE,CAAA;AAAA,UAC/C,YAAY,WAAA,CAAY,SAAA;AAAA,UACxB,aAAa,WAAA,CAAY,UAAA;AAAA,UACzB,cAAc,WAAA,CAAY,YAAA;AAAA,UAC1B,WAAW,WAAA,CAAY,QAAA,GAAW,KAAK,SAAA,CAAU,WAAA,CAAY,QAAQ,CAAA,GAAI,IAAA;AAAA,UACzE,eAAe,WAAA,CAAY,WAAA;AAAA,UAC3B,QAAQ,WAAA,CAAY,KAAA;AAAA,UACpB,UAAA,EAAA,iBAAY,IAAI,IAAA,EAAK,EAAE,WAAA;AAAY;AACrC,OACD,CAAA;AAAA,IACH;AAAA,EACF,CAAC,CAAA;AACH;AAEA,eAAsB,WAAA,GAAc;AAClC,EAAA,IAAI,OAAA,CAAQ,GAAA,CAAI,qBAAqB,CAAA,EAAG;AACtC,IAAA,MAAM,IAAI,MAAM,mEAAmE,CAAA;AAAA,EACrF;AAEA,EAAA,MAAM,WAAA,GAAc,OAAO,UAAA,EAAW;AACtC,EAAA,OAAA,CAAQ,GAAA,CAAI,qBAAqB,CAAA,GAAI,WAAA;AACvC","file":"index.cjs","sourcesContent":["export const GLOBAL_RUN_ID_ENV_KEY = '_MASTRA_GLOBAL_RUN_ID_';\n","import { evaluate as coreEvaluate } from '@mastra/core';\nimport type { Metric } from '@mastra/core';\nimport type { Agent } from '@mastra/core/agent';\n\nimport { GLOBAL_RUN_ID_ENV_KEY } from './constants';\n\nexport async function evaluate<T extends Agent>(agent: T, input: Parameters<T['generate']>[0], metric: Metric) {\n  const testInfo = await getCurrentTestInfo();\n  let globalRunId = process.env[GLOBAL_RUN_ID_ENV_KEY];\n  const runId = crypto.randomUUID();\n  const agentOutput = await agent.generate(input, {\n    runId,\n  });\n\n  if (!globalRunId) {\n    globalRunId = process.env[GLOBAL_RUN_ID_ENV_KEY] = crypto.randomUUID();\n    console.warn('Global run id not set, you should run \"globalSetup\" from \"@mastra/evals\" before evaluating.');\n  }\n\n  const metricResult = await coreEvaluate({\n    agentName: agent.name,\n    input,\n    metric,\n    output: agentOutput.text,\n    globalRunId,\n    runId,\n    testInfo,\n    instructions: agent.instructions,\n  });\n\n  return metricResult;\n}\n\nexport const getCurrentTestInfo = async () => {\n  // Jest\n  // @ts-ignore\n  if (typeof expect !== 'undefined' && expect.getState) {\n    // @ts-ignore\n    const state = expect.getState();\n    return {\n      testName: state.currentTestName,\n      testPath: state.testPath,\n    };\n  }\n\n  try {\n    const vitest = await import('vitest');\n    if (typeof vitest !== 'undefined' && vitest.expect?.getState) {\n      const state = vitest.expect.getState();\n      return {\n        testName: state.currentTestName,\n        testPath: state.testPath,\n      };\n    }\n  } catch {}\n\n  return undefined;\n};\n","import type { Mastra } from '@mastra/core';\nimport { AvailableHooks, registerHook } from '@mastra/core/hooks';\nimport { TABLE_EVALS } from '@mastra/core/storage';\nimport { checkEvalStorageFields } from '@mastra/core/utils';\n\nimport { GLOBAL_RUN_ID_ENV_KEY } from './constants';\n\nexport async function attachListeners(mastra?: Mastra) {\n  registerHook(AvailableHooks.ON_EVALUATION, async traceObject => {\n    const storage = mastra?.getStorage();\n    if (storage) {\n      // Check for required fields\n      const logger = mastra?.getLogger();\n      const areFieldsValid = checkEvalStorageFields(traceObject, logger);\n      if (!areFieldsValid) return;\n\n      await storage.insert({\n        tableName: TABLE_EVALS,\n        record: {\n          input: traceObject.input,\n          output: traceObject.output,\n          result: JSON.stringify(traceObject.result || {}),\n          agent_name: traceObject.agentName,\n          metric_name: traceObject.metricName,\n          instructions: traceObject.instructions,\n          test_info: traceObject.testInfo ? JSON.stringify(traceObject.testInfo) : null,\n          global_run_id: traceObject.globalRunId,\n          run_id: traceObject.runId,\n          created_at: new Date().toISOString(),\n        },\n      });\n    }\n  });\n}\n\nexport async function globalSetup() {\n  if (process.env[GLOBAL_RUN_ID_ENV_KEY]) {\n    throw new Error('Global run id already set, you should only run \"GlobalSetup\" once');\n  }\n\n  const globalRunId = crypto.randomUUID();\n  process.env[GLOBAL_RUN_ID_ENV_KEY] = globalRunId;\n}\n"]}
+{"version":3,"sources":["../src/constants.ts","../src/evaluation.ts","../src/attachListeners.ts"],"names":["coreEvaluate","registerHook","AvailableHooks","storage","checkEvalStorageFields","TABLE_EVALS"],"mappings":";;;;;;;;AAAO,IAAM,qBAAA,GAAwB,wBAAA;;;ACMrC,eAAsB,QAAA,CAA0B,KAAA,EAAU,KAAA,EAAqC,MAAA,EAAgB;AAC7G,EAAA,MAAM,QAAA,GAAW,MAAM,kBAAA,EAAmB;AAC1C,EAAA,IAAI,WAAA,GAAc,OAAA,CAAQ,GAAA,CAAI,qBAAqB,CAAA;AACnD,EAAA,MAAM,KAAA,GAAQ,OAAO,UAAA,EAAW;AAChC,EAAA,MAAM,WAAA,GAAc,MAAM,KAAA,CAAM,QAAA,CAAS,KAAA,EAAO;AAAA,IAC9C;AAAA,GACD,CAAA;AAED,EAAA,IAAI,CAAC,WAAA,EAAa;AAChB,IAAA,WAAA,GAAc,OAAA,CAAQ,GAAA,CAAI,qBAAqB,CAAA,GAAI,OAAO,UAAA,EAAW;AACrE,IAAA,OAAA,CAAQ,KAAK,6FAA6F,CAAA;AAAA,EAC5G;AAEA,EAAA,MAAM,YAAA,GAAe,MAAMA,cAAA,CAAa;AAAA,IACtC,WAAW,KAAA,CAAM,IAAA;AAAA,IACjB,KAAA;AAAA,IACA,MAAA;AAAA,IACA,QAAQ,WAAA,CAAY,IAAA;AAAA,IACpB,WAAA;AAAA,IACA,KAAA;AAAA,IACA,QAAA;AAAA,IACA,cAAc,KAAA,CAAM;AAAA,GACrB,CAAA;AAED,EAAA,OAAO,YAAA;AACT;AAEO,IAAM,qBAAqB,YAAY;AAG5C,EAAA,IAAI,OAAO,MAAA,KAAW,WAAA,IAAe,MAAA,CAAO,QAAA,EAAU;AAEpD,IAAA,MAAM,KAAA,GAAQ,OAAO,QAAA,EAAS;AAC9B,IAAA,OAAO;AAAA,MACL,UAAU,KAAA,CAAM,eAAA;AAAA,MAChB,UAAU,KAAA,CAAM;AAAA,KAClB;AAAA,EACF;AAEA,EAAA,IAAI;AACF,IAAA,MAAM,MAAA,GAAS,MAAM,OAAO,qBAAQ,CAAA;AACpC,IAAA,IAAI,OAAO,MAAA,KAAW,WAAA,IAAe,MAAA,CAAO,QAAQ,QAAA,EAAU;AAC5D,MAAA,MAAM,KAAA,GAAQ,MAAA,CAAO,MAAA,CAAO,QAAA,EAAS;AACrC,MAAA,OAAO;AAAA,QACL,UAAU,KAAA,CAAM,eAAA;AAAA,QAChB,UAAU,KAAA,CAAM;AAAA,OAClB;AAAA,IACF;AAAA,EACF,CAAA,CAAA,MAAQ;AAAA,EAAC;AAET,EAAA,OAAO,MAAA;AACT,CAAA;AClDA,eAAsB,gBAAgB,MAAA,EAAiB;AACrD,EAAAC,kBAAA,CAAaC,oBAAA,CAAe,aAAA,EAAe,OAAM,WAAA,KAAe;AAC9D,IAAA,MAAMC,SAAA,GAAU,QAAQ,UAAA,EAAW;AACnC,IAAA,IAAIA,SAAA,EAAS;AAEX,MAAA,MAAM,MAAA,GAAS,QAAQ,SAAA,EAAU;AACjC,MAAA,MAAM,cAAA,GAAiBC,4BAAA,CAAuB,WAAA,EAAa,MAAM,CAAA;AACjE,MAAA,IAAI,CAAC,cAAA,EAAgB;AAErB,MAAA,MAAMD,UAAQ,MAAA,CAAO;AAAA,QACnB,SAAA,EAAWE,mBAAA;AAAA,QACX,MAAA,EAAQ;AAAA,UACN,OAAO,WAAA,CAAY,KAAA;AAAA,UACnB,QAAQ,WAAA,CAAY,MAAA;AAAA,UACpB,QAAQ,IAAA,CAAK,SAAA,CAAU,WAAA,CAAY,MAAA,IAAU,EAAE,CAAA;AAAA,UAC/C,YAAY,WAAA,CAAY,SAAA;AAAA,UACxB,aAAa,WAAA,CAAY,UAAA;AAAA,UACzB,cAAc,WAAA,CAAY,YAAA;AAAA,UAC1B,WAAW,WAAA,CAAY,QAAA,GAAW,KAAK,SAAA,CAAU,WAAA,CAAY,QAAQ,CAAA,GAAI,IAAA;AAAA,UACzE,eAAe,WAAA,CAAY,WAAA;AAAA,UAC3B,QAAQ,WAAA,CAAY,KAAA;AAAA,UACpB,UAAA,EAAA,iBAAY,IAAI,IAAA,EAAK,EAAE,WAAA;AAAY;AACrC,OACD,CAAA;AAAA,IACH;AAAA,EACF,CAAC,CAAA;AACH;AAEA,eAAsB,WAAA,GAAc;AAClC,EAAA,IAAI,OAAA,CAAQ,GAAA,CAAI,qBAAqB,CAAA,EAAG;AACtC,IAAA,MAAM,IAAI,MAAM,mEAAmE,CAAA;AAAA,EACrF;AAEA,EAAA,MAAM,WAAA,GAAc,OAAO,UAAA,EAAW;AACtC,EAAA,OAAA,CAAQ,GAAA,CAAI,qBAAqB,CAAA,GAAI,WAAA;AACvC","file":"index.cjs","sourcesContent":["export const GLOBAL_RUN_ID_ENV_KEY = '_MASTRA_GLOBAL_RUN_ID_';\n","import type { Agent } from '@mastra/core/agent';\nimport { evaluate as coreEvaluate } from '@mastra/core/eval';\nimport type { Metric } from '@mastra/core/eval';\n\nimport { GLOBAL_RUN_ID_ENV_KEY } from './constants';\n\nexport async function evaluate<T extends Agent>(agent: T, input: Parameters<T['generate']>[0], metric: Metric) {\n  const testInfo = await getCurrentTestInfo();\n  let globalRunId = process.env[GLOBAL_RUN_ID_ENV_KEY];\n  const runId = crypto.randomUUID();\n  const agentOutput = await agent.generate(input, {\n    runId,\n  });\n\n  if (!globalRunId) {\n    globalRunId = process.env[GLOBAL_RUN_ID_ENV_KEY] = crypto.randomUUID();\n    console.warn('Global run id not set, you should run \"globalSetup\" from \"@mastra/evals\" before evaluating.');\n  }\n\n  const metricResult = await coreEvaluate({\n    agentName: agent.name,\n    input,\n    metric,\n    output: agentOutput.text,\n    globalRunId,\n    runId,\n    testInfo,\n    instructions: agent.instructions,\n  });\n\n  return metricResult;\n}\n\nexport const getCurrentTestInfo = async () => {\n  // Jest\n  // @ts-ignore\n  if (typeof expect !== 'undefined' && expect.getState) {\n    // @ts-ignore\n    const state = expect.getState();\n    return {\n      testName: state.currentTestName,\n      testPath: state.testPath,\n    };\n  }\n\n  try {\n    const vitest = await import('vitest');\n    if (typeof vitest !== 'undefined' && vitest.expect?.getState) {\n      const state = vitest.expect.getState();\n      return {\n        testName: state.currentTestName,\n        testPath: state.testPath,\n      };\n    }\n  } catch {}\n\n  return undefined;\n};\n","import type { Mastra } from '@mastra/core';\nimport { AvailableHooks, registerHook } from '@mastra/core/hooks';\nimport { TABLE_EVALS } from '@mastra/core/storage';\nimport { checkEvalStorageFields } from '@mastra/core/utils';\n\nimport { GLOBAL_RUN_ID_ENV_KEY } from './constants';\n\nexport async function attachListeners(mastra?: Mastra) {\n  registerHook(AvailableHooks.ON_EVALUATION, async traceObject => {\n    const storage = mastra?.getStorage();\n    if (storage) {\n      // Check for required fields\n      const logger = mastra?.getLogger();\n      const areFieldsValid = checkEvalStorageFields(traceObject, logger);\n      if (!areFieldsValid) return;\n\n      await storage.insert({\n        tableName: TABLE_EVALS,\n        record: {\n          input: traceObject.input,\n          output: traceObject.output,\n          result: JSON.stringify(traceObject.result || {}),\n          agent_name: traceObject.agentName,\n          metric_name: traceObject.metricName,\n          instructions: traceObject.instructions,\n          test_info: traceObject.testInfo ? JSON.stringify(traceObject.testInfo) : null,\n          global_run_id: traceObject.globalRunId,\n          run_id: traceObject.runId,\n          created_at: new Date().toISOString(),\n        },\n      });\n    }\n  });\n}\n\nexport async function globalSetup() {\n  if (process.env[GLOBAL_RUN_ID_ENV_KEY]) {\n    throw new Error('Global run id already set, you should only run \"GlobalSetup\" once');\n  }\n\n  const globalRunId = crypto.randomUUID();\n  process.env[GLOBAL_RUN_ID_ENV_KEY] = globalRunId;\n}\n"]}

package/dist/index.js

@@ -1,4 +1,4 @@
-import { evaluate as evaluate$1 } from '@mastra/core';
+import { evaluate as evaluate$1 } from '@mastra/core/eval';
 import { registerHook, AvailableHooks } from '@mastra/core/hooks';
 import { TABLE_EVALS } from '@mastra/core/storage';
 import { checkEvalStorageFields } from '@mastra/core/utils';
@@ -39,7 +39,7 @@ var getCurrentTestInfo = async () => {
     };
   }
   try {
-    const vitest = await import('./dist-JRG62SVA.js');
+    const vitest = await import('./dist-JVIEAZJ6.js');
     if (typeof vitest !== "undefined" && vitest.expect?.getState) {
       const state = vitest.expect.getState();
       return {

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/constants.ts","../src/evaluation.ts","../src/attachListeners.ts"],"names":["coreEvaluate"],"mappings":";;;;;;AAAO,IAAM,qBAAA,GAAwB,wBAAA;;;ACMrC,eAAsB,QAAA,CAA0B,KAAA,EAAU,KAAA,EAAqC,MAAA,EAAgB;AAC7G,EAAA,MAAM,QAAA,GAAW,MAAM,kBAAA,EAAmB;AAC1C,EAAA,IAAI,WAAA,GAAc,OAAA,CAAQ,GAAA,CAAI,qBAAqB,CAAA;AACnD,EAAA,MAAM,KAAA,GAAQ,OAAO,UAAA,EAAW;AAChC,EAAA,MAAM,WAAA,GAAc,MAAM,KAAA,CAAM,QAAA,CAAS,KAAA,EAAO;AAAA,IAC9C;AAAA,GACD,CAAA;AAED,EAAA,IAAI,CAAC,WAAA,EAAa;AAChB,IAAA,WAAA,GAAc,OAAA,CAAQ,GAAA,CAAI,qBAAqB,CAAA,GAAI,OAAO,UAAA,EAAW;AACrE,IAAA,OAAA,CAAQ,KAAK,6FAA6F,CAAA;AAAA,EAC5G;AAEA,EAAA,MAAM,YAAA,GAAe,MAAMA,UAAA,CAAa;AAAA,IACtC,WAAW,KAAA,CAAM,IAAA;AAAA,IACjB,KAAA;AAAA,IACA,MAAA;AAAA,IACA,QAAQ,WAAA,CAAY,IAAA;AAAA,IACpB,WAAA;AAAA,IACA,KAAA;AAAA,IACA,QAAA;AAAA,IACA,cAAc,KAAA,CAAM;AAAA,GACrB,CAAA;AAED,EAAA,OAAO,YAAA;AACT;AAEO,IAAM,qBAAqB,YAAY;AAG5C,EAAA,IAAI,OAAO,MAAA,KAAW,WAAA,IAAe,MAAA,CAAO,QAAA,EAAU;AAEpD,IAAA,MAAM,KAAA,GAAQ,OAAO,QAAA,EAAS;AAC9B,IAAA,OAAO;AAAA,MACL,UAAU,KAAA,CAAM,eAAA;AAAA,MAChB,UAAU,KAAA,CAAM;AAAA,KAClB;AAAA,EACF;AAEA,EAAA,IAAI;AACF,IAAA,MAAM,MAAA,GAAS,MAAM,OAAO,oBAAQ,CAAA;AACpC,IAAA,IAAI,OAAO,MAAA,KAAW,WAAA,IAAe,MAAA,CAAO,QAAQ,QAAA,EAAU;AAC5D,MAAA,MAAM,KAAA,GAAQ,MAAA,CAAO,MAAA,CAAO,QAAA,EAAS;AACrC,MAAA,OAAO;AAAA,QACL,UAAU,KAAA,CAAM,eAAA;AAAA,QAChB,UAAU,KAAA,CAAM;AAAA,OAClB;AAAA,IACF;AAAA,EACF,CAAA,CAAA,MAAQ;AAAA,EAAC;AAET,EAAA,OAAO,MAAA;AACT,CAAA;AClDA,eAAsB,gBAAgB,MAAA,EAAiB;AACrD,EAAA,YAAA,CAAa,cAAA,CAAe,aAAA,EAAe,OAAM,WAAA,KAAe;AAC9D,IAAA,MAAM,OAAA,GAAU,QAAQ,UAAA,EAAW;AACnC,IAAA,IAAI,OAAA,EAAS;AAEX,MAAA,MAAM,MAAA,GAAS,QAAQ,SAAA,EAAU;AACjC,MAAA,MAAM,cAAA,GAAiB,sBAAA,CAAuB,WAAA,EAAa,MAAM,CAAA;AACjE,MAAA,IAAI,CAAC,cAAA,EAAgB;AAErB,MAAA,MAAM,QAAQ,MAAA,CAAO;AAAA,QACnB,SAAA,EAAW,WAAA;AAAA,QACX,MAAA,EAAQ;AAAA,UACN,OAAO,WAAA,CAAY,KAAA;AAAA,UACnB,QAAQ,WAAA,CAAY,MAAA;AAAA,UACpB,QAAQ,IAAA,CAAK,SAAA,CAAU,WAAA,CAAY,MAAA,IAAU,EAAE,CAAA;AAAA,UAC/C,YAAY,WAAA,CAAY,SAAA;AAAA,UACxB,aAAa,WAAA,CAAY,UAAA;AAAA,UACzB,cAAc,WAAA,CAAY,YAAA;AAAA,UAC1B,WAAW,WAAA,CAAY,QAAA,GAAW,KAAK,SAAA,CAAU,WAAA,CAAY,QAAQ,CAAA,GAAI,IAAA;AAAA,UACzE,eAAe,WAAA,CAAY,WAAA;AAAA,UAC3B,QAAQ,WAAA,CAAY,KAAA;AAAA,UACpB,UAAA,EAAA,iBAAY,IAAI,IAAA,EAAK,EAAE,WAAA;AAAY;AACrC,OACD,CAAA;AAAA,IACH;AAAA,EACF,CAAC,CAAA;AACH;AAEA,eAAsB,WAAA,GAAc;AAClC,EAAA,IAAI,OAAA,CAAQ,GAAA,CAAI,qBAAqB,CAAA,EAAG;AACtC,IAAA,MAAM,IAAI,MAAM,mEAAmE,CAAA;AAAA,EACrF;AAEA,EAAA,MAAM,WAAA,GAAc,OAAO,UAAA,EAAW;AACtC,EAAA,OAAA,CAAQ,GAAA,CAAI,qBAAqB,CAAA,GAAI,WAAA;AACvC","file":"index.js","sourcesContent":["export const GLOBAL_RUN_ID_ENV_KEY = '_MASTRA_GLOBAL_RUN_ID_';\n","import { evaluate as coreEvaluate } from '@mastra/core';\nimport type { Metric } from '@mastra/core';\nimport type { Agent } from '@mastra/core/agent';\n\nimport { GLOBAL_RUN_ID_ENV_KEY } from './constants';\n\nexport async function evaluate<T extends Agent>(agent: T, input: Parameters<T['generate']>[0], metric: Metric) {\n  const testInfo = await getCurrentTestInfo();\n  let globalRunId = process.env[GLOBAL_RUN_ID_ENV_KEY];\n  const runId = crypto.randomUUID();\n  const agentOutput = await agent.generate(input, {\n    runId,\n  });\n\n  if (!globalRunId) {\n    globalRunId = process.env[GLOBAL_RUN_ID_ENV_KEY] = crypto.randomUUID();\n    console.warn('Global run id not set, you should run \"globalSetup\" from \"@mastra/evals\" before evaluating.');\n  }\n\n  const metricResult = await coreEvaluate({\n    agentName: agent.name,\n    input,\n    metric,\n    output: agentOutput.text,\n    globalRunId,\n    runId,\n    testInfo,\n    instructions: agent.instructions,\n  });\n\n  return metricResult;\n}\n\nexport const getCurrentTestInfo = async () => {\n  // Jest\n  // @ts-ignore\n  if (typeof expect !== 'undefined' && expect.getState) {\n    // @ts-ignore\n    const state = expect.getState();\n    return {\n      testName: state.currentTestName,\n      testPath: state.testPath,\n    };\n  }\n\n  try {\n    const vitest = await import('vitest');\n    if (typeof vitest !== 'undefined' && vitest.expect?.getState) {\n      const state = vitest.expect.getState();\n      return {\n        testName: state.currentTestName,\n        testPath: state.testPath,\n      };\n    }\n  } catch {}\n\n  return undefined;\n};\n","import type { Mastra } from '@mastra/core';\nimport { AvailableHooks, registerHook } from '@mastra/core/hooks';\nimport { TABLE_EVALS } from '@mastra/core/storage';\nimport { checkEvalStorageFields } from '@mastra/core/utils';\n\nimport { GLOBAL_RUN_ID_ENV_KEY } from './constants';\n\nexport async function attachListeners(mastra?: Mastra) {\n  registerHook(AvailableHooks.ON_EVALUATION, async traceObject => {\n    const storage = mastra?.getStorage();\n    if (storage) {\n      // Check for required fields\n      const logger = mastra?.getLogger();\n      const areFieldsValid = checkEvalStorageFields(traceObject, logger);\n      if (!areFieldsValid) return;\n\n      await storage.insert({\n        tableName: TABLE_EVALS,\n        record: {\n          input: traceObject.input,\n          output: traceObject.output,\n          result: JSON.stringify(traceObject.result || {}),\n          agent_name: traceObject.agentName,\n          metric_name: traceObject.metricName,\n          instructions: traceObject.instructions,\n          test_info: traceObject.testInfo ? JSON.stringify(traceObject.testInfo) : null,\n          global_run_id: traceObject.globalRunId,\n          run_id: traceObject.runId,\n          created_at: new Date().toISOString(),\n        },\n      });\n    }\n  });\n}\n\nexport async function globalSetup() {\n  if (process.env[GLOBAL_RUN_ID_ENV_KEY]) {\n    throw new Error('Global run id already set, you should only run \"GlobalSetup\" once');\n  }\n\n  const globalRunId = crypto.randomUUID();\n  process.env[GLOBAL_RUN_ID_ENV_KEY] = globalRunId;\n}\n"]}
+{"version":3,"sources":["../src/constants.ts","../src/evaluation.ts","../src/attachListeners.ts"],"names":["coreEvaluate"],"mappings":";;;;;;AAAO,IAAM,qBAAA,GAAwB,wBAAA;;;ACMrC,eAAsB,QAAA,CAA0B,KAAA,EAAU,KAAA,EAAqC,MAAA,EAAgB;AAC7G,EAAA,MAAM,QAAA,GAAW,MAAM,kBAAA,EAAmB;AAC1C,EAAA,IAAI,WAAA,GAAc,OAAA,CAAQ,GAAA,CAAI,qBAAqB,CAAA;AACnD,EAAA,MAAM,KAAA,GAAQ,OAAO,UAAA,EAAW;AAChC,EAAA,MAAM,WAAA,GAAc,MAAM,KAAA,CAAM,QAAA,CAAS,KAAA,EAAO;AAAA,IAC9C;AAAA,GACD,CAAA;AAED,EAAA,IAAI,CAAC,WAAA,EAAa;AAChB,IAAA,WAAA,GAAc,OAAA,CAAQ,GAAA,CAAI,qBAAqB,CAAA,GAAI,OAAO,UAAA,EAAW;AACrE,IAAA,OAAA,CAAQ,KAAK,6FAA6F,CAAA;AAAA,EAC5G;AAEA,EAAA,MAAM,YAAA,GAAe,MAAMA,UAAA,CAAa;AAAA,IACtC,WAAW,KAAA,CAAM,IAAA;AAAA,IACjB,KAAA;AAAA,IACA,MAAA;AAAA,IACA,QAAQ,WAAA,CAAY,IAAA;AAAA,IACpB,WAAA;AAAA,IACA,KAAA;AAAA,IACA,QAAA;AAAA,IACA,cAAc,KAAA,CAAM;AAAA,GACrB,CAAA;AAED,EAAA,OAAO,YAAA;AACT;AAEO,IAAM,qBAAqB,YAAY;AAG5C,EAAA,IAAI,OAAO,MAAA,KAAW,WAAA,IAAe,MAAA,CAAO,QAAA,EAAU;AAEpD,IAAA,MAAM,KAAA,GAAQ,OAAO,QAAA,EAAS;AAC9B,IAAA,OAAO;AAAA,MACL,UAAU,KAAA,CAAM,eAAA;AAAA,MAChB,UAAU,KAAA,CAAM;AAAA,KAClB;AAAA,EACF;AAEA,EAAA,IAAI;AACF,IAAA,MAAM,MAAA,GAAS,MAAM,OAAO,oBAAQ,CAAA;AACpC,IAAA,IAAI,OAAO,MAAA,KAAW,WAAA,IAAe,MAAA,CAAO,QAAQ,QAAA,EAAU;AAC5D,MAAA,MAAM,KAAA,GAAQ,MAAA,CAAO,MAAA,CAAO,QAAA,EAAS;AACrC,MAAA,OAAO;AAAA,QACL,UAAU,KAAA,CAAM,eAAA;AAAA,QAChB,UAAU,KAAA,CAAM;AAAA,OAClB;AAAA,IACF;AAAA,EACF,CAAA,CAAA,MAAQ;AAAA,EAAC;AAET,EAAA,OAAO,MAAA;AACT,CAAA;AClDA,eAAsB,gBAAgB,MAAA,EAAiB;AACrD,EAAA,YAAA,CAAa,cAAA,CAAe,aAAA,EAAe,OAAM,WAAA,KAAe;AAC9D,IAAA,MAAM,OAAA,GAAU,QAAQ,UAAA,EAAW;AACnC,IAAA,IAAI,OAAA,EAAS;AAEX,MAAA,MAAM,MAAA,GAAS,QAAQ,SAAA,EAAU;AACjC,MAAA,MAAM,cAAA,GAAiB,sBAAA,CAAuB,WAAA,EAAa,MAAM,CAAA;AACjE,MAAA,IAAI,CAAC,cAAA,EAAgB;AAErB,MAAA,MAAM,QAAQ,MAAA,CAAO;AAAA,QACnB,SAAA,EAAW,WAAA;AAAA,QACX,MAAA,EAAQ;AAAA,UACN,OAAO,WAAA,CAAY,KAAA;AAAA,UACnB,QAAQ,WAAA,CAAY,MAAA;AAAA,UACpB,QAAQ,IAAA,CAAK,SAAA,CAAU,WAAA,CAAY,MAAA,IAAU,EAAE,CAAA;AAAA,UAC/C,YAAY,WAAA,CAAY,SAAA;AAAA,UACxB,aAAa,WAAA,CAAY,UAAA;AAAA,UACzB,cAAc,WAAA,CAAY,YAAA;AAAA,UAC1B,WAAW,WAAA,CAAY,QAAA,GAAW,KAAK,SAAA,CAAU,WAAA,CAAY,QAAQ,CAAA,GAAI,IAAA;AAAA,UACzE,eAAe,WAAA,CAAY,WAAA;AAAA,UAC3B,QAAQ,WAAA,CAAY,KAAA;AAAA,UACpB,UAAA,EAAA,iBAAY,IAAI,IAAA,EAAK,EAAE,WAAA;AAAY;AACrC,OACD,CAAA;AAAA,IACH;AAAA,EACF,CAAC,CAAA;AACH;AAEA,eAAsB,WAAA,GAAc;AAClC,EAAA,IAAI,OAAA,CAAQ,GAAA,CAAI,qBAAqB,CAAA,EAAG;AACtC,IAAA,MAAM,IAAI,MAAM,mEAAmE,CAAA;AAAA,EACrF;AAEA,EAAA,MAAM,WAAA,GAAc,OAAO,UAAA,EAAW;AACtC,EAAA,OAAA,CAAQ,GAAA,CAAI,qBAAqB,CAAA,GAAI,WAAA;AACvC","file":"index.js","sourcesContent":["export const GLOBAL_RUN_ID_ENV_KEY = '_MASTRA_GLOBAL_RUN_ID_';\n","import type { Agent } from '@mastra/core/agent';\nimport { evaluate as coreEvaluate } from '@mastra/core/eval';\nimport type { Metric } from '@mastra/core/eval';\n\nimport { GLOBAL_RUN_ID_ENV_KEY } from './constants';\n\nexport async function evaluate<T extends Agent>(agent: T, input: Parameters<T['generate']>[0], metric: Metric) {\n  const testInfo = await getCurrentTestInfo();\n  let globalRunId = process.env[GLOBAL_RUN_ID_ENV_KEY];\n  const runId = crypto.randomUUID();\n  const agentOutput = await agent.generate(input, {\n    runId,\n  });\n\n  if (!globalRunId) {\n    globalRunId = process.env[GLOBAL_RUN_ID_ENV_KEY] = crypto.randomUUID();\n    console.warn('Global run id not set, you should run \"globalSetup\" from \"@mastra/evals\" before evaluating.');\n  }\n\n  const metricResult = await coreEvaluate({\n    agentName: agent.name,\n    input,\n    metric,\n    output: agentOutput.text,\n    globalRunId,\n    runId,\n    testInfo,\n    instructions: agent.instructions,\n  });\n\n  return metricResult;\n}\n\nexport const getCurrentTestInfo = async () => {\n  // Jest\n  // @ts-ignore\n  if (typeof expect !== 'undefined' && expect.getState) {\n    // @ts-ignore\n    const state = expect.getState();\n    return {\n      testName: state.currentTestName,\n      testPath: state.testPath,\n    };\n  }\n\n  try {\n    const vitest = await import('vitest');\n    if (typeof vitest !== 'undefined' && vitest.expect?.getState) {\n      const state = vitest.expect.getState();\n      return {\n        testName: state.currentTestName,\n        testPath: state.testPath,\n      };\n    }\n  } catch {}\n\n  return undefined;\n};\n","import type { Mastra } from '@mastra/core';\nimport { AvailableHooks, registerHook } from '@mastra/core/hooks';\nimport { TABLE_EVALS } from '@mastra/core/storage';\nimport { checkEvalStorageFields } from '@mastra/core/utils';\n\nimport { GLOBAL_RUN_ID_ENV_KEY } from './constants';\n\nexport async function attachListeners(mastra?: Mastra) {\n  registerHook(AvailableHooks.ON_EVALUATION, async traceObject => {\n    const storage = mastra?.getStorage();\n    if (storage) {\n      // Check for required fields\n      const logger = mastra?.getLogger();\n      const areFieldsValid = checkEvalStorageFields(traceObject, logger);\n      if (!areFieldsValid) return;\n\n      await storage.insert({\n        tableName: TABLE_EVALS,\n        record: {\n          input: traceObject.input,\n          output: traceObject.output,\n          result: JSON.stringify(traceObject.result || {}),\n          agent_name: traceObject.agentName,\n          metric_name: traceObject.metricName,\n          instructions: traceObject.instructions,\n          test_info: traceObject.testInfo ? JSON.stringify(traceObject.testInfo) : null,\n          global_run_id: traceObject.globalRunId,\n          run_id: traceObject.runId,\n          created_at: new Date().toISOString(),\n        },\n      });\n    }\n  });\n}\n\nexport async function globalSetup() {\n  if (process.env[GLOBAL_RUN_ID_ENV_KEY]) {\n    throw new Error('Global run id already set, you should only run \"GlobalSetup\" once');\n  }\n\n  const globalRunId = crypto.randomUUID();\n  process.env[GLOBAL_RUN_ID_ENV_KEY] = globalRunId;\n}\n"]}

package/dist/scorers/code/index.cjs

@@ -1,5 +1,6 @@
 'use strict';
 
+var chunkQVZBKGOE_cjs = require('../../chunk-QVZBKGOE.cjs');
 var scores = require('@mastra/core/scores');
 var nlp = require('compromise');
 var difflib = require('difflib');
@@ -228,11 +229,95 @@ function createToneScorer(config = {}) {
     return results.preprocessStepResult?.score;
   });
 }
+function checkToolOrder(actualTools, expectedOrder, strictMode = false) {
+  if (strictMode) {
+    return JSON.stringify(actualTools) === JSON.stringify(expectedOrder);
+  }
+  const expectedIndices = [];
+  for (const expectedTool of expectedOrder) {
+    const index = actualTools.indexOf(expectedTool);
+    if (index === -1) {
+      return false;
+    }
+    expectedIndices.push(index);
+  }
+  for (let i = 1; i < expectedIndices.length; i++) {
+    const currentIndex = expectedIndices[i];
+    const prevIndex = expectedIndices[i - 1];
+    if (currentIndex !== void 0 && prevIndex !== void 0 && currentIndex <= prevIndex) {
+      return false;
+    }
+  }
+  return true;
+}
+function calculateAccuracy({
+  expectedTool,
+  actualTools,
+  strictMode = false,
+  expectedToolOrder
+}) {
+  if (actualTools.length === 0) {
+    return 0;
+  }
+  if (expectedToolOrder && expectedToolOrder.length > 0) {
+    return checkToolOrder(actualTools, expectedToolOrder, strictMode) ? 1 : 0;
+  }
+  if (!expectedTool) {
+    return 0;
+  }
+  if (strictMode) {
+    return actualTools.length === 1 && actualTools[0] === expectedTool ? 1 : 0;
+  }
+  return actualTools.includes(expectedTool) ? 1 : 0;
+}
+function createToolCallAccuracyScorerCode(options) {
+  const { expectedTool, strictMode = false, expectedToolOrder } = options;
+  if (!expectedTool && !expectedToolOrder) {
+    throw new Error("Either expectedTool or expectedToolOrder must be provided");
+  }
+  const getDescription = () => {
+    return expectedToolOrder ? `Evaluates whether the LLM called tools in the correct order: [${expectedToolOrder.join(", ")}]` : `Evaluates whether the LLM selected the correct tool (${expectedTool}) from the available tools`;
+  };
+  return scores.createScorer({
+    name: "Tool Call Accuracy",
+    description: getDescription()
+  }).preprocess(async ({ run }) => {
+    const isInputInvalid = !run.input || !run.input.inputMessages || run.input.inputMessages.length === 0;
+    const isOutputInvalid = !run.output || run.output.length === 0;
+    if (isInputInvalid || isOutputInvalid) {
+      throw new Error("Input and output messages cannot be null or empty");
+    }
+    const { tools: actualTools, toolCallInfos } = chunkQVZBKGOE_cjs.extractToolCalls(run.output);
+    const correctToolCalled = expectedTool ? strictMode ? actualTools.length === 1 && actualTools[0] === expectedTool : actualTools.includes(expectedTool) : false;
+    return {
+      expectedTool,
+      actualTools,
+      strictMode,
+      expectedToolOrder,
+      hasToolCalls: actualTools.length > 0,
+      correctToolCalled,
+      toolCallInfos,
+      correctOrderCalled: expectedToolOrder ? checkToolOrder(actualTools, expectedToolOrder, strictMode) : null
+    };
+  }).generateScore(({ results }) => {
+    const preprocessResult = results.preprocessStepResult;
+    if (!preprocessResult) {
+      return 0;
+    }
+    return calculateAccuracy({
+      expectedTool: preprocessResult.expectedTool,
+      actualTools: preprocessResult.actualTools,
+      strictMode: preprocessResult.strictMode,
+      expectedToolOrder: preprocessResult.expectedToolOrder
+    });
+  });
+}
 
 exports.createCompletenessScorer = createCompletenessScorer;
 exports.createContentSimilarityScorer = createContentSimilarityScorer;
 exports.createKeywordCoverageScorer = createKeywordCoverageScorer;
 exports.createTextualDifferenceScorer = createTextualDifferenceScorer;
 exports.createToneScorer = createToneScorer;
+exports.createToolCallAccuracyScorerCode = createToolCallAccuracyScorerCode;
 //# sourceMappingURL=index.cjs.map
 //# sourceMappingURL=index.cjs.map

package/dist/scorers/code/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/scorers/code/completeness/index.ts","../../../src/scorers/code/textual-difference/index.ts","../../../src/scorers/code/keyword-coverage/index.ts","../../../src/scorers/code/content-similarity/index.ts","../../../src/scorers/code/tone/index.ts"],"names":["createScorer","nlp","SequenceMatcher","keyword_extractor","stringSimilarity","Sentiment"],"mappings":";;;;;;;;;;;;;;;;AAIA,SAAS,gBAAgB,GAAA,EAAqB;AAE5C,EAAA,OAAO,GAAA,CACJ,UAAU,KAAK,CAAA,CACf,QAAQ,kBAAA,EAAoB,EAAE,EAC9B,WAAA,EAAY;AACjB;AAEA,SAAS,gBAAgB,GAAA,EAAoB;AAE3C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAC3C,EAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAM,CAAE,cAAa,CAAE,GAAA,CAAI,OAAO,CAAA,IAAK,EAAC;AAC1D,EAAA,MAAM,SAAS,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,OAAO,KAAK,EAAC;AAC7C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAG3C,EAAA,MAAM,iBAAA,GAAoB,CAAC,IAAA,KAA2B;AAEpD,IAAA,MAAM,UAAA,GAAa,gBAAgB,IAAI,CAAA;AAGvC,IAAA,OAAO,WACJ,OAAA,CAAQ,iBAAA,EAAmB,OAAO,CAAA,CAClC,OAAA,CAAQ,eAAe,GAAG,CAAA,CAC1B,IAAA,EAAK,CACL,MAAM,KAAK,CAAA,CACX,OAAO,CAAA,IAAA,KAAQ,IAAA,CAAK,SAAS,CAAC,CAAA;AAAA,EACnC,CAAA;AAGA,EAAA,MAAM,cAAA,GAAiB;AAAA,IACrB,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,MAAA,CAAO,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IACnC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB;AAAA,GACpC;AAGA,EAAA,OAAO,CAAC,GAAG,IAAI,GAAA,CAAI,cAAc,CAAC,CAAA;AACpC;AAEA,SAAS,iBAAA,CAAkB,EAAE,QAAA,EAAU,UAAA,EAAW,EAAyD;AACzG,EAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,IAAA,OAAO,UAAA,CAAW,MAAA,KAAW,CAAA,GAAI,CAAA,GAAI,CAAA;AAAA,EACvC;AAGA,EAAA,MAAM,UAAU,QAAA,CAAS,MAAA;AAAA,IAAO,CAAA,OAAA,KAC9B,UAAA,CAAW,IAAA,CAAK,CAAA,CAAA,KAAK;AACnB,MAAA,MAAM,IAAA,GAAO,gBAAgB,OAAO,CAAA;AACpC,MAAA,MAAM,IAAA,GAAO,gBAAgB,CAAC,CAAA;AAG9B,MAAA,IAAI,IAAA,CAAK,UAAU,CAAA,EAAG;AACpB,QAAA,OAAO,IAAA,KAAS,IAAA;AAAA,MAClB;AAGA,MAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAClD,MAAA,MAAM,OAAA,GAAU,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAEnD,MAAA,IAAI,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC5B,QAAA,OAAO,OAAA,CAAQ,MAAA,GAAS,MAAA,CAAO,MAAA,GAAS,GAAA;AAAA,MAC1C;AAEA,MAAA,OAAO,KAAA;AAAA,IACT,CAAC;AAAA,GACH;AACA,EAAA,OAAO,OAAA,CAAQ,SAAS,QAAA,CAAS,MAAA;AACnC;AAEO,SAAS,wBAAA,GAA2B;AACzC,EAAA,OAAOA,mBAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GACJ,CAAC,GAAA,CAAI,KAAA,IACL,IAAI,KAAA,CAAM,aAAA,CAAc,IAAA,CAAK,CAAC,MAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAExG,IAAA,MAAM,eAAA,GACJ,CAAC,GAAA,CAAI,MAAA,IAAU,IAAI,MAAA,CAAO,IAAA,CAAK,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAE1G,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,oCAAoC,CAAA;AAAA,IACtD;AAEA,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAChG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,EAAE,OAAA,EAAQ,KAA2B,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE5F,IAAA,MAAM,cAAA,GAAiB,KAAA;AACvB,IAAA,MAAM,eAAA,GAAkB,MAAA;AAExB,IAAA,MAAM,QAAA,GAAWC,oBAAA,CAAI,cAAA,CAAe,IAAA,EAAM,CAAA;AAC1C,IAAA,MAAM,SAAA,GAAYA,oBAAA,CAAI,eAAA,CAAgB,IAAA,EAAM,CAAA;AAG5C,IAAA,MAAM,aAAA,GAAgB,gBAAgB,QAAQ,CAAA;AAC9C,IAAA,MAAM,cAAA,GAAiB,gBAAgB,SAAS,CAAA;AAEhD,IAAA,OAAO;AAAA,MACL,aAAA;AAAA,MACA,cAAA;AAAA,MACA,eAAA,EAAiB,cAAc,MAAA,CAAO,CAAA,CAAA,KAAK,CAAC,cAAA,CAAe,QAAA,CAAS,CAAC,CAAC,CAAA;AAAA,MACtE,aAAA,EAAe;AAAA,QACb,OAAO,aAAA,CAAc,MAAA;AAAA,QACrB,QAAQ,cAAA,CAAe;AAAA;AACzB,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAA,GAAgB,QAAQ,oBAAA,EAAsB,aAAA;AACpD,IAAA,MAAM,cAAA,GAAiB,QAAQ,oBAAA,EAAsB,cAAA;AAErD,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,QAAA,EAAU,aAAA;AAAA,MACV,UAAA,EAAY;AAAA,KACb,CAAA;AAAA,EACH,CAAC,CAAA;AACL;ACzHO,SAAS,6BAAA,GAAgC;AAC9C,EAAA,OAAOD,mBAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACpF,IAAA,MAAM,OAAA,GAAU,IAAIE,uBAAA,CAAgB,IAAA,EAAM,OAAO,MAAM,CAAA;AACvD,IAAA,MAAM,KAAA,GAAQ,QAAQ,KAAA,EAAM;AAG5B,IAAA,MAAM,GAAA,GAAM,QAAQ,UAAA,EAAW;AAC/B,IAAA,MAAM,OAAA,GAAU,IAAI,MAAA,CAAO,CAAC,CAAC,EAAE,CAAA,KAAM,EAAA,KAAO,OAAO,CAAA,CAAE,MAAA;AAGrD,IAAA,MAAM,YAAY,IAAA,CAAK,GAAA,CAAI,KAAA,CAAM,MAAA,EAAQ,OAAO,MAAM,CAAA;AACtD,IAAA,MAAM,UAAA,GAAa,SAAA,GAAY,CAAA,GAAI,IAAA,CAAK,GAAA,CAAI,MAAM,MAAA,GAAS,MAAA,CAAO,MAAM,CAAA,GAAI,SAAA,GAAY,CAAA;AACxF,IAAA,MAAM,aAAa,CAAA,GAAI,UAAA;AAEvB,IAAA,OAAO;AAAA,MACL,KAAA;AAAA,MACA,UAAA;AAAA,MACA,OAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;AC/BO,SAAS,2BAAA,GAA8B;AAC5C,EAAA,OAAOF,mBAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAEpF,IAAA,IAAI,CAAC,KAAA,IAAS,CAAC,MAAA,EAAQ;AACrB,MAAA,OAAO;AAAA,QACL,MAAA,EAAQ;AAAA,UACN,iBAAA,sBAAuB,GAAA,EAAY;AAAA,UACnC,gBAAA,sBAAsB,GAAA;AAAY;AACpC,OACF;AAAA,IACF;AAEA,IAAA,MAAM,eAAA,GAAkB,CAAC,IAAA,KAAiB;AACxC,MAAA,OAAOG,kCAAA,CAAkB,QAAQ,IAAA,EAAM;AAAA,QACrC,QAAA,EAAU,SAAA;AAAA,QACV,aAAA,EAAe,IAAA;AAAA,QACf,mBAAA,EAAqB,IAAA;AAAA,QACrB,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAAA,IACH,CAAA;AAEA,IAAA,MAAM,iBAAA,GAAoB,IAAI,GAAA,CAAI,eAAA,CAAgB,KAAK,CAAC,CAAA;AACxD,IAAA,MAAM,gBAAA,GAAmB,IAAI,GAAA,CAAI,eAAA,CAAgB,MAAM,CAAC,CAAA;AACxD,IAAA,OAAO;AAAA,MACL,iBAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,OAAA,CAAQ,OAAO,EAAE,SAAQ,KAAM;AAC9B,IAAA,IACE,CAAC,QAAQ,oBAAA,EAAsB,iBAAA,EAAmB,QAClD,CAAC,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAA,EACjD;AACA,MAAA,OAAO;AAAA,QACL,mBAAA,EAAqB,CAAA;AAAA,QACrB,qBAAA,EAAuB;AAAA,OACzB;AAAA,IACF;AAEA,IAAA,MAAM,kBAAkB,CAAC,GAAG,OAAA,CAAQ,oBAAA,EAAsB,iBAAiB,CAAA,CAAE,MAAA;AAAA,MAAO,CAAA,CAAA,KAClF,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAI,CAAC;AAAA,KACvD;AAEA,IAAA,OAAO;AAAA,MACL,qBAAqB,KAAA,CAAM,IAAA,CAAK,QAAQ,oBAAA,EAAsB,iBAAiB,EAAE,MAAA,IAAU,CAAA;AAAA,MAC3F,qBAAA,EAAuB,gBAAgB,MAAA,IAAU;AAAA,KACnD;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,IAAI,CAAC,OAAA,CAAQ,iBAAA,EAAmB,mBAAA,EAAqB;AACnD,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,MAAM,aAAA,GAAgB,QAAQ,iBAAA,EAAmB,mBAAA;AACjD,IAAA,MAAM,eAAA,GAAkB,QAAQ,iBAAA,EAAmB,qBAAA;AACnD,IAAA,OAAO,aAAA,GAAgB,CAAA,GAAI,eAAA,GAAkB,aAAA,GAAgB,CAAA;AAAA,EAC/D,CAAC,CAAA;AACL;AC3DO,SAAS,6BAAA,CACd,EAAE,UAAA,EAAY,gBAAA,EAAiB,GAA8B,EAAE,UAAA,EAAY,IAAA,EAAM,gBAAA,EAAkB,IAAA,EAAK,EACxG;AACA,EAAA,OAAOH,mBAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,IAAI,cAAA,GAAiB,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACvG,IAAA,IAAI,eAAA,GAAkB,GAAA,CAAI,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE1F,IAAA,IAAI,UAAA,EAAY;AACd,MAAA,cAAA,GAAiB,eAAe,WAAA,EAAY;AAC5C,MAAA,eAAA,GAAkB,gBAAgB,WAAA,EAAY;AAAA,IAChD;AAEA,IAAA,IAAI,gBAAA,EAAkB;AACpB,MAAA,cAAA,GAAiB,cAAA,CAAe,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAC1D,MAAA,eAAA,GAAkB,eAAA,CAAgB,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAAA,IAC9D;AAEA,IAAA,OAAO;AAAA,MACL,cAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAaI,iCAAA,CAAiB,iBAAA;AAAA,MAClC,QAAQ,oBAAA,EAAsB,cAAA;AAAA,MAC9B,QAAQ,oBAAA,EAAsB;AAAA,KAChC;AAEA,IAAA,OAAO,UAAA;AAAA,EACT,CAAC,CAAA;AACL;ACpCO,SAAS,gBAAA,CAAiB,MAAA,GAA2B,EAAC,EAAG;AAC9D,EAAA,MAAM,EAAE,eAAc,GAAI,MAAA;AAE1B,EAAA,OAAOJ,mBAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,SAAA,GAAY,IAAIK,0BAAA,EAAU;AAChC,IAAA,MAAM,YAAA,GAAuB,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAClG,IAAA,MAAM,iBAAA,GAAoB,SAAA,CAAU,OAAA,CAAQ,YAAY,CAAA;AAExD,IAAA,IAAI,aAAA,EAAe;AAEjB,MAAA,MAAM,kBAAA,GAAqB,SAAA,CAAU,OAAA,CAAQ,aAAa,CAAA;AAC1D,MAAA,MAAM,gBAAgB,IAAA,CAAK,GAAA,CAAI,iBAAA,CAAkB,WAAA,GAAc,mBAAmB,WAAW,CAAA;AAC7F,MAAA,MAAM,eAAA,GAAkB,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,aAAa,CAAA;AAErD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,eAAA;AAAA,QACP,mBAAmB,iBAAA,CAAkB,WAAA;AAAA,QACrC,oBAAoB,kBAAA,CAAmB,WAAA;AAAA,QACvC,UAAA,EAAY;AAAA,OACd;AAAA,IACF;AAGA,IAAA,MAAM,YAAY,YAAA,CAAa,KAAA,CAAM,gBAAgB,CAAA,IAAK,CAAC,YAAY,CAAA;AACvE,IAAA,MAAM,UAAA,GAAa,UAAU,GAAA,CAAI,CAAA,CAAA,KAAK,UAAU,OAAA,CAAQ,CAAC,EAAE,WAAW,CAAA;AACtE,IAAA,MAAM,YAAA,GAAe,UAAA,CAAW,MAAA,CAAO,CAAC,CAAA,EAAG,MAAM,CAAA,GAAI,CAAA,EAAG,CAAC,CAAA,GAAI,UAAA,CAAW,MAAA;AACxE,IAAA,MAAM,QAAA,GAAW,UAAA,CAAW,MAAA,CAAO,CAAC,KAAK,CAAA,KAAM,GAAA,GAAM,IAAA,CAAK,GAAA,CAAI,IAAI,YAAA,EAAc,CAAC,CAAA,EAAG,CAAC,IAAI,UAAA,CAAW,MAAA;AACpG,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,QAAQ,CAAA;AAE1C,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,SAAA;AAAA,MACP,YAAA;AAAA,MACA,iBAAA,EAAmB;AAAA,KACrB;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL","file":"index.cjs","sourcesContent":["import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport nlp from 'compromise';\n\nfunction normalizeString(str: string): string {\n  // Remove diacritics and convert to lowercase\n  return str\n    .normalize('NFD')\n    .replace(/[\\u0300-\\u036f]/g, '')\n    .toLowerCase();\n}\n\nfunction extractElements(doc: any): string[] {\n  // Get more specific elements and ensure they're arrays\n  const nouns = doc.nouns().out('array') || [];\n  const verbs = doc.verbs().toInfinitive().out('array') || [];\n  const topics = doc.topics().out('array') || [];\n  const terms = doc.terms().out('array') || [];\n\n  // Helper function to clean and split terms\n  const cleanAndSplitTerm = (term: string): string[] => {\n    // First normalize the string\n    const normalized = normalizeString(term);\n\n    // Split on word boundaries and filter out empty strings\n    return normalized\n      .replace(/([a-z])([A-Z])/g, '$1 $2') // Split camelCase\n      .replace(/[^a-z0-9]+/g, ' ') // Replace non-alphanumeric with spaces\n      .trim()\n      .split(/\\s+/)\n      .filter(word => word.length > 0);\n  };\n\n  // Process all elements\n  const processedTerms = [\n    ...nouns.flatMap(cleanAndSplitTerm),\n    ...verbs.flatMap(cleanAndSplitTerm),\n    ...topics.flatMap(cleanAndSplitTerm),\n    ...terms.flatMap(cleanAndSplitTerm),\n  ];\n\n  // Remove duplicates\n  return [...new Set(processedTerms)];\n}\n\nfunction calculateCoverage({ original, simplified }: { original: string[]; simplified: string[] }): number {\n  if (original.length === 0) {\n    return simplified.length === 0 ? 1 : 0;\n  }\n\n  // Exact matching for short words (3 chars or less), substring matching for longer words\n  const covered = original.filter(element =>\n    simplified.some(s => {\n      const elem = normalizeString(element);\n      const simp = normalizeString(s);\n\n      // For short words (3 chars or less), require exact match\n      if (elem.length <= 3) {\n        return elem === simp;\n      }\n\n      // For longer words, require substantial overlap (more than 60% of the longer word)\n      const longer = elem.length > simp.length ? elem : simp;\n      const shorter = elem.length > simp.length ? simp : elem;\n\n      if (longer.includes(shorter)) {\n        return shorter.length / longer.length > 0.6;\n      }\n\n      return false;\n    }),\n  );\n  return covered.length / original.length;\n}\n\nexport function createCompletenessScorer() {\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      const isInputInvalid =\n        !run.input ||\n        run.input.inputMessages.some((i: { content: string }) => i.content === null || i.content === undefined);\n\n      const isOutputInvalid =\n        !run.output || run.output.some((i: { content: string }) => i.content === null || i.content === undefined);\n\n      if (isInputInvalid || isOutputInvalid) {\n        throw new Error('Inputs cannot be null or undefined');\n      }\n\n      const input = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map(({ content }: { content: string }) => content).join(', ') || '';\n\n      const inputToProcess = input;\n      const outputToProcess = output;\n\n      const inputDoc = nlp(inputToProcess.trim());\n      const outputDoc = nlp(outputToProcess.trim());\n\n      // Extract and log elements\n      const inputElements = extractElements(inputDoc);\n      const outputElements = extractElements(outputDoc);\n\n      return {\n        inputElements,\n        outputElements,\n        missingElements: inputElements.filter(e => !outputElements.includes(e)),\n        elementCounts: {\n          input: inputElements.length,\n          output: outputElements.length,\n        },\n      };\n    })\n    .generateScore(({ results }) => {\n      const inputElements = results.preprocessStepResult?.inputElements;\n      const outputElements = results.preprocessStepResult?.outputElements;\n\n      return calculateCoverage({\n        original: inputElements,\n        simplified: outputElements,\n      });\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport { SequenceMatcher } from 'difflib';\n\nexport function createTextualDifferenceScorer() {\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n      const matcher = new SequenceMatcher(null, input, output);\n      const ratio = matcher.ratio();\n\n      // Get detailed operations\n      const ops = matcher.getOpcodes();\n      const changes = ops.filter(([op]) => op !== 'equal').length;\n\n      // Calculate confidence based on text length difference\n      const maxLength = Math.max(input.length, output.length);\n      const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;\n      const confidence = 1 - lengthDiff;\n\n      return {\n        ratio,\n        confidence,\n        changes,\n        lengthDiff,\n      };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.ratio;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport keyword_extractor from 'keyword-extractor';\n\nexport function createKeywordCoverageScorer() {\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n\n      if (!input && !output) {\n        return {\n          result: {\n            referenceKeywords: new Set<string>(),\n            responseKeywords: new Set<string>(),\n          },\n        };\n      }\n\n      const extractKeywords = (text: string) => {\n        return keyword_extractor.extract(text, {\n          language: 'english',\n          remove_digits: true,\n          return_changed_case: true,\n          remove_duplicates: true,\n        });\n      };\n\n      const referenceKeywords = new Set(extractKeywords(input));\n      const responseKeywords = new Set(extractKeywords(output));\n      return {\n        referenceKeywords,\n        responseKeywords,\n      };\n    })\n    .analyze(async ({ results }) => {\n      if (\n        !results.preprocessStepResult?.referenceKeywords?.size &&\n        !results.preprocessStepResult?.responseKeywords?.size\n      ) {\n        return {\n          totalKeywordsLength: 0,\n          matchedKeywordsLength: 0,\n        };\n      }\n\n      const matchedKeywords = [...results.preprocessStepResult?.referenceKeywords].filter(k =>\n        results.preprocessStepResult?.responseKeywords?.has(k),\n      );\n\n      return {\n        totalKeywordsLength: Array.from(results.preprocessStepResult?.referenceKeywords).length ?? 0,\n        matchedKeywordsLength: matchedKeywords.length ?? 0,\n      };\n    })\n    .generateScore(({ results }) => {\n      if (!results.analyzeStepResult?.totalKeywordsLength) {\n        return 1;\n      }\n\n      const totalKeywords = results.analyzeStepResult?.totalKeywordsLength!;\n      const matchedKeywords = results.analyzeStepResult?.matchedKeywordsLength!;\n      return totalKeywords > 0 ? matchedKeywords / totalKeywords : 0;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport stringSimilarity from 'string-similarity';\n\ninterface ContentSimilarityOptions {\n  ignoreCase?: boolean;\n  ignoreWhitespace?: boolean;\n}\n\nexport function createContentSimilarityScorer(\n  { ignoreCase, ignoreWhitespace }: ContentSimilarityOptions = { ignoreCase: true, ignoreWhitespace: true },\n) {\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      let processedInput = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') || '';\n      let processedOutput = run.output.map((i: { content: string }) => i.content).join(', ') || '';\n\n      if (ignoreCase) {\n        processedInput = processedInput.toLowerCase();\n        processedOutput = processedOutput.toLowerCase();\n      }\n\n      if (ignoreWhitespace) {\n        processedInput = processedInput.replace(/\\s+/g, ' ').trim();\n        processedOutput = processedOutput.replace(/\\s+/g, ' ').trim();\n      }\n\n      return {\n        processedInput,\n        processedOutput,\n      };\n    })\n    .generateScore(({ results }) => {\n      const similarity = stringSimilarity.compareTwoStrings(\n        results.preprocessStepResult?.processedInput,\n        results.preprocessStepResult?.processedOutput,\n      );\n\n      return similarity;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport Sentiment from 'sentiment';\n\ninterface ToneScorerConfig {\n  referenceTone?: string;\n}\n\nexport function createToneScorer(config: ToneScorerConfig = {}) {\n  const { referenceTone } = config;\n\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      const sentiment = new Sentiment();\n      const agentMessage: string = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n      const responseSentiment = sentiment.analyze(agentMessage);\n\n      if (referenceTone) {\n        // Compare sentiment with reference\n        const referenceSentiment = sentiment.analyze(referenceTone);\n        const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);\n        const normalizedScore = Math.max(0, 1 - sentimentDiff);\n\n        return {\n          score: normalizedScore,\n          responseSentiment: responseSentiment.comparative,\n          referenceSentiment: referenceSentiment.comparative,\n          difference: sentimentDiff,\n        };\n      }\n\n      // Evaluate sentiment stability across response\n      const sentences = agentMessage.match(/[^.!?]+[.!?]+/g) || [agentMessage];\n      const sentiments = sentences.map(s => sentiment.analyze(s).comparative);\n      const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;\n      const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;\n      const stability = Math.max(0, 1 - variance);\n\n      return {\n        score: stability,\n        avgSentiment,\n        sentimentVariance: variance,\n      };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.score;\n    });\n}\n"]}
+{"version":3,"sources":["../../../src/scorers/code/completeness/index.ts","../../../src/scorers/code/textual-difference/index.ts","../../../src/scorers/code/keyword-coverage/index.ts","../../../src/scorers/code/content-similarity/index.ts","../../../src/scorers/code/tone/index.ts","../../../src/scorers/code/tool-call-accuracy/index.ts"],"names":["createScorer","nlp","SequenceMatcher","keyword_extractor","stringSimilarity","Sentiment","extractToolCalls"],"mappings":";;;;;;;;;;;;;;;;;AAIA,SAAS,gBAAgB,GAAA,EAAqB;AAE5C,EAAA,OAAO,GAAA,CACJ,UAAU,KAAK,CAAA,CACf,QAAQ,kBAAA,EAAoB,EAAE,EAC9B,WAAA,EAAY;AACjB;AAEA,SAAS,gBAAgB,GAAA,EAAoB;AAE3C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAC3C,EAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAM,CAAE,cAAa,CAAE,GAAA,CAAI,OAAO,CAAA,IAAK,EAAC;AAC1D,EAAA,MAAM,SAAS,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,OAAO,KAAK,EAAC;AAC7C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAG3C,EAAA,MAAM,iBAAA,GAAoB,CAAC,IAAA,KAA2B;AAEpD,IAAA,MAAM,UAAA,GAAa,gBAAgB,IAAI,CAAA;AAGvC,IAAA,OAAO,WACJ,OAAA,CAAQ,iBAAA,EAAmB,OAAO,CAAA,CAClC,OAAA,CAAQ,eAAe,GAAG,CAAA,CAC1B,IAAA,EAAK,CACL,MAAM,KAAK,CAAA,CACX,OAAO,CAAA,IAAA,KAAQ,IAAA,CAAK,SAAS,CAAC,CAAA;AAAA,EACnC,CAAA;AAGA,EAAA,MAAM,cAAA,GAAiB;AAAA,IACrB,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,MAAA,CAAO,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IACnC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB;AAAA,GACpC;AAGA,EAAA,OAAO,CAAC,GAAG,IAAI,GAAA,CAAI,cAAc,CAAC,CAAA;AACpC;AAEA,SAAS,iBAAA,CAAkB,EAAE,QAAA,EAAU,UAAA,EAAW,EAAyD;AACzG,EAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,IAAA,OAAO,UAAA,CAAW,MAAA,KAAW,CAAA,GAAI,CAAA,GAAI,CAAA;AAAA,EACvC;AAGA,EAAA,MAAM,UAAU,QAAA,CAAS,MAAA;AAAA,IAAO,CAAA,OAAA,KAC9B,UAAA,CAAW,IAAA,CAAK,CAAA,CAAA,KAAK;AACnB,MAAA,MAAM,IAAA,GAAO,gBAAgB,OAAO,CAAA;AACpC,MAAA,MAAM,IAAA,GAAO,gBAAgB,CAAC,CAAA;AAG9B,MAAA,IAAI,IAAA,CAAK,UAAU,CAAA,EAAG;AACpB,QAAA,OAAO,IAAA,KAAS,IAAA;AAAA,MAClB;AAGA,MAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAClD,MAAA,MAAM,OAAA,GAAU,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAEnD,MAAA,IAAI,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC5B,QAAA,OAAO,OAAA,CAAQ,MAAA,GAAS,MAAA,CAAO,MAAA,GAAS,GAAA;AAAA,MAC1C;AAEA,MAAA,OAAO,KAAA;AAAA,IACT,CAAC;AAAA,GACH;AACA,EAAA,OAAO,OAAA,CAAQ,SAAS,QAAA,CAAS,MAAA;AACnC;AAEO,SAAS,wBAAA,GAA2B;AACzC,EAAA,OAAOA,mBAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GACJ,CAAC,GAAA,CAAI,KAAA,IACL,IAAI,KAAA,CAAM,aAAA,CAAc,IAAA,CAAK,CAAC,MAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAExG,IAAA,MAAM,eAAA,GACJ,CAAC,GAAA,CAAI,MAAA,IAAU,IAAI,MAAA,CAAO,IAAA,CAAK,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAE1G,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,oCAAoC,CAAA;AAAA,IACtD;AAEA,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAChG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,EAAE,OAAA,EAAQ,KAA2B,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE5F,IAAA,MAAM,cAAA,GAAiB,KAAA;AACvB,IAAA,MAAM,eAAA,GAAkB,MAAA;AAExB,IAAA,MAAM,QAAA,GAAWC,oBAAA,CAAI,cAAA,CAAe,IAAA,EAAM,CAAA;AAC1C,IAAA,MAAM,SAAA,GAAYA,oBAAA,CAAI,eAAA,CAAgB,IAAA,EAAM,CAAA;AAG5C,IAAA,MAAM,aAAA,GAAgB,gBAAgB,QAAQ,CAAA;AAC9C,IAAA,MAAM,cAAA,GAAiB,gBAAgB,SAAS,CAAA;AAEhD,IAAA,OAAO;AAAA,MACL,aAAA;AAAA,MACA,cAAA;AAAA,MACA,eAAA,EAAiB,cAAc,MAAA,CAAO,CAAA,CAAA,KAAK,CAAC,cAAA,CAAe,QAAA,CAAS,CAAC,CAAC,CAAA;AAAA,MACtE,aAAA,EAAe;AAAA,QACb,OAAO,aAAA,CAAc,MAAA;AAAA,QACrB,QAAQ,cAAA,CAAe;AAAA;AACzB,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAA,GAAgB,QAAQ,oBAAA,EAAsB,aAAA;AACpD,IAAA,MAAM,cAAA,GAAiB,QAAQ,oBAAA,EAAsB,cAAA;AAErD,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,QAAA,EAAU,aAAA;AAAA,MACV,UAAA,EAAY;AAAA,KACb,CAAA;AAAA,EACH,CAAC,CAAA;AACL;ACzHO,SAAS,6BAAA,GAAgC;AAC9C,EAAA,OAAOD,mBAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACpF,IAAA,MAAM,OAAA,GAAU,IAAIE,uBAAA,CAAgB,IAAA,EAAM,OAAO,MAAM,CAAA;AACvD,IAAA,MAAM,KAAA,GAAQ,QAAQ,KAAA,EAAM;AAG5B,IAAA,MAAM,GAAA,GAAM,QAAQ,UAAA,EAAW;AAC/B,IAAA,MAAM,OAAA,GAAU,IAAI,MAAA,CAAO,CAAC,CAAC,EAAE,CAAA,KAAM,EAAA,KAAO,OAAO,CAAA,CAAE,MAAA;AAGrD,IAAA,MAAM,YAAY,IAAA,CAAK,GAAA,CAAI,KAAA,CAAM,MAAA,EAAQ,OAAO,MAAM,CAAA;AACtD,IAAA,MAAM,UAAA,GAAa,SAAA,GAAY,CAAA,GAAI,IAAA,CAAK,GAAA,CAAI,MAAM,MAAA,GAAS,MAAA,CAAO,MAAM,CAAA,GAAI,SAAA,GAAY,CAAA;AACxF,IAAA,MAAM,aAAa,CAAA,GAAI,UAAA;AAEvB,IAAA,OAAO;AAAA,MACL,KAAA;AAAA,MACA,UAAA;AAAA,MACA,OAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;AC/BO,SAAS,2BAAA,GAA8B;AAC5C,EAAA,OAAOF,mBAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAEpF,IAAA,IAAI,CAAC,KAAA,IAAS,CAAC,MAAA,EAAQ;AACrB,MAAA,OAAO;AAAA,QACL,MAAA,EAAQ;AAAA,UACN,iBAAA,sBAAuB,GAAA,EAAY;AAAA,UACnC,gBAAA,sBAAsB,GAAA;AAAY;AACpC,OACF;AAAA,IACF;AAEA,IAAA,MAAM,eAAA,GAAkB,CAAC,IAAA,KAAiB;AACxC,MAAA,OAAOG,kCAAA,CAAkB,QAAQ,IAAA,EAAM;AAAA,QACrC,QAAA,EAAU,SAAA;AAAA,QACV,aAAA,EAAe,IAAA;AAAA,QACf,mBAAA,EAAqB,IAAA;AAAA,QACrB,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAAA,IACH,CAAA;AAEA,IAAA,MAAM,iBAAA,GAAoB,IAAI,GAAA,CAAI,eAAA,CAAgB,KAAK,CAAC,CAAA;AACxD,IAAA,MAAM,gBAAA,GAAmB,IAAI,GAAA,CAAI,eAAA,CAAgB,MAAM,CAAC,CAAA;AACxD,IAAA,OAAO;AAAA,MACL,iBAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,OAAA,CAAQ,OAAO,EAAE,SAAQ,KAAM;AAC9B,IAAA,IACE,CAAC,QAAQ,oBAAA,EAAsB,iBAAA,EAAmB,QAClD,CAAC,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAA,EACjD;AACA,MAAA,OAAO;AAAA,QACL,mBAAA,EAAqB,CAAA;AAAA,QACrB,qBAAA,EAAuB;AAAA,OACzB;AAAA,IACF;AAEA,IAAA,MAAM,kBAAkB,CAAC,GAAG,OAAA,CAAQ,oBAAA,EAAsB,iBAAiB,CAAA,CAAE,MAAA;AAAA,MAAO,CAAA,CAAA,KAClF,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAI,CAAC;AAAA,KACvD;AAEA,IAAA,OAAO;AAAA,MACL,qBAAqB,KAAA,CAAM,IAAA,CAAK,QAAQ,oBAAA,EAAsB,iBAAiB,EAAE,MAAA,IAAU,CAAA;AAAA,MAC3F,qBAAA,EAAuB,gBAAgB,MAAA,IAAU;AAAA,KACnD;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,IAAI,CAAC,OAAA,CAAQ,iBAAA,EAAmB,mBAAA,EAAqB;AACnD,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,MAAM,aAAA,GAAgB,QAAQ,iBAAA,EAAmB,mBAAA;AACjD,IAAA,MAAM,eAAA,GAAkB,QAAQ,iBAAA,EAAmB,qBAAA;AACnD,IAAA,OAAO,aAAA,GAAgB,CAAA,GAAI,eAAA,GAAkB,aAAA,GAAgB,CAAA;AAAA,EAC/D,CAAC,CAAA;AACL;AC3DO,SAAS,6BAAA,CACd,EAAE,UAAA,EAAY,gBAAA,EAAiB,GAA8B,EAAE,UAAA,EAAY,IAAA,EAAM,gBAAA,EAAkB,IAAA,EAAK,EACxG;AACA,EAAA,OAAOH,mBAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,IAAI,cAAA,GAAiB,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACvG,IAAA,IAAI,eAAA,GAAkB,GAAA,CAAI,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE1F,IAAA,IAAI,UAAA,EAAY;AACd,MAAA,cAAA,GAAiB,eAAe,WAAA,EAAY;AAC5C,MAAA,eAAA,GAAkB,gBAAgB,WAAA,EAAY;AAAA,IAChD;AAEA,IAAA,IAAI,gBAAA,EAAkB;AACpB,MAAA,cAAA,GAAiB,cAAA,CAAe,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAC1D,MAAA,eAAA,GAAkB,eAAA,CAAgB,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAAA,IAC9D;AAEA,IAAA,OAAO;AAAA,MACL,cAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAaI,iCAAA,CAAiB,iBAAA;AAAA,MAClC,QAAQ,oBAAA,EAAsB,cAAA;AAAA,MAC9B,QAAQ,oBAAA,EAAsB;AAAA,KAChC;AAEA,IAAA,OAAO,UAAA;AAAA,EACT,CAAC,CAAA;AACL;ACpCO,SAAS,gBAAA,CAAiB,MAAA,GAA2B,EAAC,EAAG;AAC9D,EAAA,MAAM,EAAE,eAAc,GAAI,MAAA;AAE1B,EAAA,OAAOJ,mBAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,SAAA,GAAY,IAAIK,0BAAA,EAAU;AAChC,IAAA,MAAM,YAAA,GAAuB,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAClG,IAAA,MAAM,iBAAA,GAAoB,SAAA,CAAU,OAAA,CAAQ,YAAY,CAAA;AAExD,IAAA,IAAI,aAAA,EAAe;AAEjB,MAAA,MAAM,kBAAA,GAAqB,SAAA,CAAU,OAAA,CAAQ,aAAa,CAAA;AAC1D,MAAA,MAAM,gBAAgB,IAAA,CAAK,GAAA,CAAI,iBAAA,CAAkB,WAAA,GAAc,mBAAmB,WAAW,CAAA;AAC7F,MAAA,MAAM,eAAA,GAAkB,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,aAAa,CAAA;AAErD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,eAAA;AAAA,QACP,mBAAmB,iBAAA,CAAkB,WAAA;AAAA,QACrC,oBAAoB,kBAAA,CAAmB,WAAA;AAAA,QACvC,UAAA,EAAY;AAAA,OACd;AAAA,IACF;AAGA,IAAA,MAAM,YAAY,YAAA,CAAa,KAAA,CAAM,gBAAgB,CAAA,IAAK,CAAC,YAAY,CAAA;AACvE,IAAA,MAAM,UAAA,GAAa,UAAU,GAAA,CAAI,CAAA,CAAA,KAAK,UAAU,OAAA,CAAQ,CAAC,EAAE,WAAW,CAAA;AACtE,IAAA,MAAM,YAAA,GAAe,UAAA,CAAW,MAAA,CAAO,CAAC,CAAA,EAAG,MAAM,CAAA,GAAI,CAAA,EAAG,CAAC,CAAA,GAAI,UAAA,CAAW,MAAA;AACxE,IAAA,MAAM,QAAA,GAAW,UAAA,CAAW,MAAA,CAAO,CAAC,KAAK,CAAA,KAAM,GAAA,GAAM,IAAA,CAAK,GAAA,CAAI,IAAI,YAAA,EAAc,CAAC,CAAA,EAAG,CAAC,IAAI,UAAA,CAAW,MAAA;AACpG,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,QAAQ,CAAA;AAE1C,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,SAAA;AAAA,MACP,YAAA;AAAA,MACA,iBAAA,EAAmB;AAAA,KACrB;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;ACzCA,SAAS,cAAA,CAAe,WAAA,EAAuB,aAAA,EAAyB,UAAA,GAAsB,KAAA,EAAgB;AAC5G,EAAA,IAAI,UAAA,EAAY;AACd,IAAA,OAAO,KAAK,SAAA,CAAU,WAAW,CAAA,KAAM,IAAA,CAAK,UAAU,aAAa,CAAA;AAAA,EACrE;AAEA,EAAA,MAAM,kBAA4B,EAAC;AACnC,EAAA,KAAA,MAAW,gBAAgB,aAAA,EAAe;AACxC,IAAA,MAAM,KAAA,GAAQ,WAAA,CAAY,OAAA,CAAQ,YAAY,CAAA;AAC9C,IAAA,IAAI,UAAU,EAAA,EAAI;AAChB,MAAA,OAAO,KAAA;AAAA,IACT;AACA,IAAA,eAAA,CAAgB,KAAK,KAAK,CAAA;AAAA,EAC5B;AAEA,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,eAAA,CAAgB,QAAQ,CAAA,EAAA,EAAK;AAC/C,IAAA,MAAM,YAAA,GAAe,gBAAgB,CAAC,CAAA;AACtC,IAAA,MAAM,SAAA,GAAY,eAAA,CAAgB,CAAA,GAAI,CAAC,CAAA;AACvC,IAAA,IAAI,YAAA,KAAiB,MAAA,IAAa,SAAA,KAAc,MAAA,IAAa,gBAAgB,SAAA,EAAW;AACtF,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AAEA,EAAA,OAAO,IAAA;AACT;AAEA,SAAS,iBAAA,CAAkB;AAAA,EACzB,YAAA;AAAA,EACA,WAAA;AAAA,EACA,UAAA,GAAa,KAAA;AAAA,EACb;AACF,CAAA,EAKW;AACT,EAAA,IAAI,WAAA,CAAY,WAAW,CAAA,EAAG;AAC5B,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,IAAI,iBAAA,IAAqB,iBAAA,CAAkB,MAAA,GAAS,CAAA,EAAG;AACrD,IAAA,OAAO,cAAA,CAAe,WAAA,EAAa,iBAAA,EAAmB,UAAU,IAAI,CAAA,GAAI,CAAA;AAAA,EAC1E;AAEA,EAAA,IAAI,CAAC,YAAA,EAAc;AACjB,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,IAAI,UAAA,EAAY;AACd,IAAA,OAAO,YAAY,MAAA,KAAW,CAAA,IAAK,YAAY,CAAC,CAAA,KAAM,eAAe,CAAA,GAAI,CAAA;AAAA,EAC3E;AAEA,EAAA,OAAO,WAAA,CAAY,QAAA,CAAS,YAAY,CAAA,GAAI,CAAA,GAAI,CAAA;AAClD;AAEO,SAAS,iCAAiC,OAAA,EAAkC;AACjF,EAAA,MAAM,EAAE,YAAA,EAAc,UAAA,GAAa,KAAA,EAAO,mBAAkB,GAAI,OAAA;AAEhE,EAAA,IAAI,CAAC,YAAA,IAAgB,CAAC,iBAAA,EAAmB;AACvC,IAAA,MAAM,IAAI,MAAM,2DAA2D,CAAA;AAAA,EAC7E;AAEA,EAAA,MAAM,iBAAiB,MAAM;AAC3B,IAAA,OAAO,iBAAA,GACH,iEAAiE,iBAAA,CAAkB,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA,CAAA,GAC7F,wDAAwD,YAAY,CAAA,0BAAA,CAAA;AAAA,EAC1E,CAAA;AAEA,EAAA,OAAOL,mBAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,oBAAA;AAAA,IACN,aAAa,cAAA;AAAe,GAC7B,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GAAiB,CAAC,GAAA,CAAI,KAAA,IAAS,CAAC,GAAA,CAAI,KAAA,CAAM,aAAA,IAAiB,GAAA,CAAI,KAAA,CAAM,aAAA,CAAc,MAAA,KAAW,CAAA;AACpG,IAAA,MAAM,kBAAkB,CAAC,GAAA,CAAI,MAAA,IAAU,GAAA,CAAI,OAAO,MAAA,KAAW,CAAA;AAE7D,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,mDAAmD,CAAA;AAAA,IACrE;AAEA,IAAA,MAAM,EAAE,KAAA,EAAO,WAAA,EAAa,eAAc,GAAIM,kCAAA,CAAiB,IAAI,MAAM,CAAA;AAEzE,IAAA,MAAM,iBAAA,GAAoB,YAAA,GACtB,UAAA,GACE,WAAA,CAAY,MAAA,KAAW,CAAA,IAAK,WAAA,CAAY,CAAC,CAAA,KAAM,YAAA,GAC/C,WAAA,CAAY,QAAA,CAAS,YAAY,CAAA,GACnC,KAAA;AAEJ,IAAA,OAAO;AAAA,MACL,YAAA;AAAA,MACA,WAAA;AAAA,MACA,UAAA;AAAA,MACA,iBAAA;AAAA,MACA,YAAA,EAAc,YAAY,MAAA,GAAS,CAAA;AAAA,MACnC,iBAAA;AAAA,MACA,aAAA;AAAA,MACA,oBAAoB,iBAAA,GAAoB,cAAA,CAAe,WAAA,EAAa,iBAAA,EAAmB,UAAU,CAAA,GAAI;AAAA,KACvG;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,mBAAmB,OAAA,CAAQ,oBAAA;AACjC,IAAA,IAAI,CAAC,gBAAA,EAAkB;AACrB,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,cAAc,gBAAA,CAAiB,YAAA;AAAA,MAC/B,aAAa,gBAAA,CAAiB,WAAA;AAAA,MAC9B,YAAY,gBAAA,CAAiB,UAAA;AAAA,MAC7B,mBAAmB,gBAAA,CAAiB;AAAA,KACrC,CAAA;AAAA,EACH,CAAC,CAAA;AACL","file":"index.cjs","sourcesContent":["import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport nlp from 'compromise';\n\nfunction normalizeString(str: string): string {\n  // Remove diacritics and convert to lowercase\n  return str\n    .normalize('NFD')\n    .replace(/[\\u0300-\\u036f]/g, '')\n    .toLowerCase();\n}\n\nfunction extractElements(doc: any): string[] {\n  // Get more specific elements and ensure they're arrays\n  const nouns = doc.nouns().out('array') || [];\n  const verbs = doc.verbs().toInfinitive().out('array') || [];\n  const topics = doc.topics().out('array') || [];\n  const terms = doc.terms().out('array') || [];\n\n  // Helper function to clean and split terms\n  const cleanAndSplitTerm = (term: string): string[] => {\n    // First normalize the string\n    const normalized = normalizeString(term);\n\n    // Split on word boundaries and filter out empty strings\n    return normalized\n      .replace(/([a-z])([A-Z])/g, '$1 $2') // Split camelCase\n      .replace(/[^a-z0-9]+/g, ' ') // Replace non-alphanumeric with spaces\n      .trim()\n      .split(/\\s+/)\n      .filter(word => word.length > 0);\n  };\n\n  // Process all elements\n  const processedTerms = [\n    ...nouns.flatMap(cleanAndSplitTerm),\n    ...verbs.flatMap(cleanAndSplitTerm),\n    ...topics.flatMap(cleanAndSplitTerm),\n    ...terms.flatMap(cleanAndSplitTerm),\n  ];\n\n  // Remove duplicates\n  return [...new Set(processedTerms)];\n}\n\nfunction calculateCoverage({ original, simplified }: { original: string[]; simplified: string[] }): number {\n  if (original.length === 0) {\n    return simplified.length === 0 ? 1 : 0;\n  }\n\n  // Exact matching for short words (3 chars or less), substring matching for longer words\n  const covered = original.filter(element =>\n    simplified.some(s => {\n      const elem = normalizeString(element);\n      const simp = normalizeString(s);\n\n      // For short words (3 chars or less), require exact match\n      if (elem.length <= 3) {\n        return elem === simp;\n      }\n\n      // For longer words, require substantial overlap (more than 60% of the longer word)\n      const longer = elem.length > simp.length ? elem : simp;\n      const shorter = elem.length > simp.length ? simp : elem;\n\n      if (longer.includes(shorter)) {\n        return shorter.length / longer.length > 0.6;\n      }\n\n      return false;\n    }),\n  );\n  return covered.length / original.length;\n}\n\nexport function createCompletenessScorer() {\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      const isInputInvalid =\n        !run.input ||\n        run.input.inputMessages.some((i: { content: string }) => i.content === null || i.content === undefined);\n\n      const isOutputInvalid =\n        !run.output || run.output.some((i: { content: string }) => i.content === null || i.content === undefined);\n\n      if (isInputInvalid || isOutputInvalid) {\n        throw new Error('Inputs cannot be null or undefined');\n      }\n\n      const input = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map(({ content }: { content: string }) => content).join(', ') || '';\n\n      const inputToProcess = input;\n      const outputToProcess = output;\n\n      const inputDoc = nlp(inputToProcess.trim());\n      const outputDoc = nlp(outputToProcess.trim());\n\n      // Extract and log elements\n      const inputElements = extractElements(inputDoc);\n      const outputElements = extractElements(outputDoc);\n\n      return {\n        inputElements,\n        outputElements,\n        missingElements: inputElements.filter(e => !outputElements.includes(e)),\n        elementCounts: {\n          input: inputElements.length,\n          output: outputElements.length,\n        },\n      };\n    })\n    .generateScore(({ results }) => {\n      const inputElements = results.preprocessStepResult?.inputElements;\n      const outputElements = results.preprocessStepResult?.outputElements;\n\n      return calculateCoverage({\n        original: inputElements,\n        simplified: outputElements,\n      });\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport { SequenceMatcher } from 'difflib';\n\nexport function createTextualDifferenceScorer() {\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n      const matcher = new SequenceMatcher(null, input, output);\n      const ratio = matcher.ratio();\n\n      // Get detailed operations\n      const ops = matcher.getOpcodes();\n      const changes = ops.filter(([op]) => op !== 'equal').length;\n\n      // Calculate confidence based on text length difference\n      const maxLength = Math.max(input.length, output.length);\n      const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;\n      const confidence = 1 - lengthDiff;\n\n      return {\n        ratio,\n        confidence,\n        changes,\n        lengthDiff,\n      };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.ratio;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport keyword_extractor from 'keyword-extractor';\n\nexport function createKeywordCoverageScorer() {\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n\n      if (!input && !output) {\n        return {\n          result: {\n            referenceKeywords: new Set<string>(),\n            responseKeywords: new Set<string>(),\n          },\n        };\n      }\n\n      const extractKeywords = (text: string) => {\n        return keyword_extractor.extract(text, {\n          language: 'english',\n          remove_digits: true,\n          return_changed_case: true,\n          remove_duplicates: true,\n        });\n      };\n\n      const referenceKeywords = new Set(extractKeywords(input));\n      const responseKeywords = new Set(extractKeywords(output));\n      return {\n        referenceKeywords,\n        responseKeywords,\n      };\n    })\n    .analyze(async ({ results }) => {\n      if (\n        !results.preprocessStepResult?.referenceKeywords?.size &&\n        !results.preprocessStepResult?.responseKeywords?.size\n      ) {\n        return {\n          totalKeywordsLength: 0,\n          matchedKeywordsLength: 0,\n        };\n      }\n\n      const matchedKeywords = [...results.preprocessStepResult?.referenceKeywords].filter(k =>\n        results.preprocessStepResult?.responseKeywords?.has(k),\n      );\n\n      return {\n        totalKeywordsLength: Array.from(results.preprocessStepResult?.referenceKeywords).length ?? 0,\n        matchedKeywordsLength: matchedKeywords.length ?? 0,\n      };\n    })\n    .generateScore(({ results }) => {\n      if (!results.analyzeStepResult?.totalKeywordsLength) {\n        return 1;\n      }\n\n      const totalKeywords = results.analyzeStepResult?.totalKeywordsLength!;\n      const matchedKeywords = results.analyzeStepResult?.matchedKeywordsLength!;\n      return totalKeywords > 0 ? matchedKeywords / totalKeywords : 0;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport stringSimilarity from 'string-similarity';\n\ninterface ContentSimilarityOptions {\n  ignoreCase?: boolean;\n  ignoreWhitespace?: boolean;\n}\n\nexport function createContentSimilarityScorer(\n  { ignoreCase, ignoreWhitespace }: ContentSimilarityOptions = { ignoreCase: true, ignoreWhitespace: true },\n) {\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      let processedInput = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') || '';\n      let processedOutput = run.output.map((i: { content: string }) => i.content).join(', ') || '';\n\n      if (ignoreCase) {\n        processedInput = processedInput.toLowerCase();\n        processedOutput = processedOutput.toLowerCase();\n      }\n\n      if (ignoreWhitespace) {\n        processedInput = processedInput.replace(/\\s+/g, ' ').trim();\n        processedOutput = processedOutput.replace(/\\s+/g, ' ').trim();\n      }\n\n      return {\n        processedInput,\n        processedOutput,\n      };\n    })\n    .generateScore(({ results }) => {\n      const similarity = stringSimilarity.compareTwoStrings(\n        results.preprocessStepResult?.processedInput,\n        results.preprocessStepResult?.processedOutput,\n      );\n\n      return similarity;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport Sentiment from 'sentiment';\n\ninterface ToneScorerConfig {\n  referenceTone?: string;\n}\n\nexport function createToneScorer(config: ToneScorerConfig = {}) {\n  const { referenceTone } = config;\n\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      const sentiment = new Sentiment();\n      const agentMessage: string = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n      const responseSentiment = sentiment.analyze(agentMessage);\n\n      if (referenceTone) {\n        // Compare sentiment with reference\n        const referenceSentiment = sentiment.analyze(referenceTone);\n        const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);\n        const normalizedScore = Math.max(0, 1 - sentimentDiff);\n\n        return {\n          score: normalizedScore,\n          responseSentiment: responseSentiment.comparative,\n          referenceSentiment: referenceSentiment.comparative,\n          difference: sentimentDiff,\n        };\n      }\n\n      // Evaluate sentiment stability across response\n      const sentences = agentMessage.match(/[^.!?]+[.!?]+/g) || [agentMessage];\n      const sentiments = sentences.map(s => sentiment.analyze(s).comparative);\n      const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;\n      const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;\n      const stability = Math.max(0, 1 - variance);\n\n      return {\n        score: stability,\n        avgSentiment,\n        sentimentVariance: variance,\n      };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.score;\n    });\n}\n","import type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport { createScorer } from '@mastra/core/scores';\nimport { extractToolCalls } from '../../utils';\n\ninterface ToolCallAccuracyOptions {\n  expectedTool?: string;\n  strictMode?: boolean;\n  expectedToolOrder?: string[];\n}\n\nfunction checkToolOrder(actualTools: string[], expectedOrder: string[], strictMode: boolean = false): boolean {\n  if (strictMode) {\n    return JSON.stringify(actualTools) === JSON.stringify(expectedOrder);\n  }\n\n  const expectedIndices: number[] = [];\n  for (const expectedTool of expectedOrder) {\n    const index = actualTools.indexOf(expectedTool);\n    if (index === -1) {\n      return false;\n    }\n    expectedIndices.push(index);\n  }\n\n  for (let i = 1; i < expectedIndices.length; i++) {\n    const currentIndex = expectedIndices[i];\n    const prevIndex = expectedIndices[i - 1];\n    if (currentIndex !== undefined && prevIndex !== undefined && currentIndex <= prevIndex) {\n      return false;\n    }\n  }\n\n  return true;\n}\n\nfunction calculateAccuracy({\n  expectedTool,\n  actualTools,\n  strictMode = false,\n  expectedToolOrder,\n}: {\n  expectedTool?: string;\n  actualTools: string[];\n  strictMode?: boolean;\n  expectedToolOrder?: string[];\n}): number {\n  if (actualTools.length === 0) {\n    return 0;\n  }\n\n  if (expectedToolOrder && expectedToolOrder.length > 0) {\n    return checkToolOrder(actualTools, expectedToolOrder, strictMode) ? 1 : 0;\n  }\n\n  if (!expectedTool) {\n    return 0;\n  }\n\n  if (strictMode) {\n    return actualTools.length === 1 && actualTools[0] === expectedTool ? 1 : 0;\n  }\n\n  return actualTools.includes(expectedTool) ? 1 : 0;\n}\n\nexport function createToolCallAccuracyScorerCode(options: ToolCallAccuracyOptions) {\n  const { expectedTool, strictMode = false, expectedToolOrder } = options;\n\n  if (!expectedTool && !expectedToolOrder) {\n    throw new Error('Either expectedTool or expectedToolOrder must be provided');\n  }\n\n  const getDescription = () => {\n    return expectedToolOrder\n      ? `Evaluates whether the LLM called tools in the correct order: [${expectedToolOrder.join(', ')}]`\n      : `Evaluates whether the LLM selected the correct tool (${expectedTool}) from the available tools`;\n  };\n\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Tool Call Accuracy',\n    description: getDescription(),\n  })\n    .preprocess(async ({ run }) => {\n      const isInputInvalid = !run.input || !run.input.inputMessages || run.input.inputMessages.length === 0;\n      const isOutputInvalid = !run.output || run.output.length === 0;\n\n      if (isInputInvalid || isOutputInvalid) {\n        throw new Error('Input and output messages cannot be null or empty');\n      }\n\n      const { tools: actualTools, toolCallInfos } = extractToolCalls(run.output);\n\n      const correctToolCalled = expectedTool\n        ? strictMode\n          ? actualTools.length === 1 && actualTools[0] === expectedTool\n          : actualTools.includes(expectedTool)\n        : false;\n\n      return {\n        expectedTool,\n        actualTools,\n        strictMode,\n        expectedToolOrder,\n        hasToolCalls: actualTools.length > 0,\n        correctToolCalled,\n        toolCallInfos,\n        correctOrderCalled: expectedToolOrder ? checkToolOrder(actualTools, expectedToolOrder, strictMode) : null,\n      };\n    })\n    .generateScore(({ results }) => {\n      const preprocessResult = results.preprocessStepResult;\n      if (!preprocessResult) {\n        return 0;\n      }\n\n      return calculateAccuracy({\n        expectedTool: preprocessResult.expectedTool,\n        actualTools: preprocessResult.actualTools,\n        strictMode: preprocessResult.strictMode,\n        expectedToolOrder: preprocessResult.expectedToolOrder,\n      });\n    });\n}\n"]}

package/dist/scorers/code/index.d.ts

@@ -3,4 +3,5 @@ export * from './textual-difference/index.js';
 export * from './keyword-coverage/index.js';
 export * from './content-similarity/index.js';
 export * from './tone/index.js';
+export * from './tool-call-accuracy/index.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/scorers/code/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/scorers/code/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,sBAAsB,CAAC;AACrC,cAAc,QAAQ,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/scorers/code/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,sBAAsB,CAAC;AACrC,cAAc,QAAQ,CAAC;AACvB,cAAc,sBAAsB,CAAC"}

package/dist/scorers/code/index.js

@@ -1,3 +1,4 @@
+import { extractToolCalls } from '../../chunk-5CVZXIFW.js';
 import { createScorer } from '@mastra/core/scores';
 import nlp from 'compromise';
 import { SequenceMatcher } from 'difflib';
@@ -219,7 +220,90 @@ function createToneScorer(config = {}) {
     return results.preprocessStepResult?.score;
   });
 }
+function checkToolOrder(actualTools, expectedOrder, strictMode = false) {
+  if (strictMode) {
+    return JSON.stringify(actualTools) === JSON.stringify(expectedOrder);
+  }
+  const expectedIndices = [];
+  for (const expectedTool of expectedOrder) {
+    const index = actualTools.indexOf(expectedTool);
+    if (index === -1) {
+      return false;
+    }
+    expectedIndices.push(index);
+  }
+  for (let i = 1; i < expectedIndices.length; i++) {
+    const currentIndex = expectedIndices[i];
+    const prevIndex = expectedIndices[i - 1];
+    if (currentIndex !== void 0 && prevIndex !== void 0 && currentIndex <= prevIndex) {
+      return false;
+    }
+  }
+  return true;
+}
+function calculateAccuracy({
+  expectedTool,
+  actualTools,
+  strictMode = false,
+  expectedToolOrder
+}) {
+  if (actualTools.length === 0) {
+    return 0;
+  }
+  if (expectedToolOrder && expectedToolOrder.length > 0) {
+    return checkToolOrder(actualTools, expectedToolOrder, strictMode) ? 1 : 0;
+  }
+  if (!expectedTool) {
+    return 0;
+  }
+  if (strictMode) {
+    return actualTools.length === 1 && actualTools[0] === expectedTool ? 1 : 0;
+  }
+  return actualTools.includes(expectedTool) ? 1 : 0;
+}
+function createToolCallAccuracyScorerCode(options) {
+  const { expectedTool, strictMode = false, expectedToolOrder } = options;
+  if (!expectedTool && !expectedToolOrder) {
+    throw new Error("Either expectedTool or expectedToolOrder must be provided");
+  }
+  const getDescription = () => {
+    return expectedToolOrder ? `Evaluates whether the LLM called tools in the correct order: [${expectedToolOrder.join(", ")}]` : `Evaluates whether the LLM selected the correct tool (${expectedTool}) from the available tools`;
+  };
+  return createScorer({
+    name: "Tool Call Accuracy",
+    description: getDescription()
+  }).preprocess(async ({ run }) => {
+    const isInputInvalid = !run.input || !run.input.inputMessages || run.input.inputMessages.length === 0;
+    const isOutputInvalid = !run.output || run.output.length === 0;
+    if (isInputInvalid || isOutputInvalid) {
+      throw new Error("Input and output messages cannot be null or empty");
+    }
+    const { tools: actualTools, toolCallInfos } = extractToolCalls(run.output);
+    const correctToolCalled = expectedTool ? strictMode ? actualTools.length === 1 && actualTools[0] === expectedTool : actualTools.includes(expectedTool) : false;
+    return {
+      expectedTool,
+      actualTools,
+      strictMode,
+      expectedToolOrder,
+      hasToolCalls: actualTools.length > 0,
+      correctToolCalled,
+      toolCallInfos,
+      correctOrderCalled: expectedToolOrder ? checkToolOrder(actualTools, expectedToolOrder, strictMode) : null
+    };
+  }).generateScore(({ results }) => {
+    const preprocessResult = results.preprocessStepResult;
+    if (!preprocessResult) {
+      return 0;
+    }
+    return calculateAccuracy({
+      expectedTool: preprocessResult.expectedTool,
+      actualTools: preprocessResult.actualTools,
+      strictMode: preprocessResult.strictMode,
+      expectedToolOrder: preprocessResult.expectedToolOrder
+    });
+  });
+}
 
-export { createCompletenessScorer, createContentSimilarityScorer, createKeywordCoverageScorer, createTextualDifferenceScorer, createToneScorer };
+export { createCompletenessScorer, createContentSimilarityScorer, createKeywordCoverageScorer, createTextualDifferenceScorer, createToneScorer, createToolCallAccuracyScorerCode };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map

package/dist/scorers/code/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/scorers/code/completeness/index.ts","../../../src/scorers/code/textual-difference/index.ts","../../../src/scorers/code/keyword-coverage/index.ts","../../../src/scorers/code/content-similarity/index.ts","../../../src/scorers/code/tone/index.ts"],"names":["createScorer"],"mappings":";;;;;;;AAIA,SAAS,gBAAgB,GAAA,EAAqB;AAE5C,EAAA,OAAO,GAAA,CACJ,UAAU,KAAK,CAAA,CACf,QAAQ,kBAAA,EAAoB,EAAE,EAC9B,WAAA,EAAY;AACjB;AAEA,SAAS,gBAAgB,GAAA,EAAoB;AAE3C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAC3C,EAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAM,CAAE,cAAa,CAAE,GAAA,CAAI,OAAO,CAAA,IAAK,EAAC;AAC1D,EAAA,MAAM,SAAS,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,OAAO,KAAK,EAAC;AAC7C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAG3C,EAAA,MAAM,iBAAA,GAAoB,CAAC,IAAA,KAA2B;AAEpD,IAAA,MAAM,UAAA,GAAa,gBAAgB,IAAI,CAAA;AAGvC,IAAA,OAAO,WACJ,OAAA,CAAQ,iBAAA,EAAmB,OAAO,CAAA,CAClC,OAAA,CAAQ,eAAe,GAAG,CAAA,CAC1B,IAAA,EAAK,CACL,MAAM,KAAK,CAAA,CACX,OAAO,CAAA,IAAA,KAAQ,IAAA,CAAK,SAAS,CAAC,CAAA;AAAA,EACnC,CAAA;AAGA,EAAA,MAAM,cAAA,GAAiB;AAAA,IACrB,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,MAAA,CAAO,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IACnC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB;AAAA,GACpC;AAGA,EAAA,OAAO,CAAC,GAAG,IAAI,GAAA,CAAI,cAAc,CAAC,CAAA;AACpC;AAEA,SAAS,iBAAA,CAAkB,EAAE,QAAA,EAAU,UAAA,EAAW,EAAyD;AACzG,EAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,IAAA,OAAO,UAAA,CAAW,MAAA,KAAW,CAAA,GAAI,CAAA,GAAI,CAAA;AAAA,EACvC;AAGA,EAAA,MAAM,UAAU,QAAA,CAAS,MAAA;AAAA,IAAO,CAAA,OAAA,KAC9B,UAAA,CAAW,IAAA,CAAK,CAAA,CAAA,KAAK;AACnB,MAAA,MAAM,IAAA,GAAO,gBAAgB,OAAO,CAAA;AACpC,MAAA,MAAM,IAAA,GAAO,gBAAgB,CAAC,CAAA;AAG9B,MAAA,IAAI,IAAA,CAAK,UAAU,CAAA,EAAG;AACpB,QAAA,OAAO,IAAA,KAAS,IAAA;AAAA,MAClB;AAGA,MAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAClD,MAAA,MAAM,OAAA,GAAU,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAEnD,MAAA,IAAI,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC5B,QAAA,OAAO,OAAA,CAAQ,MAAA,GAAS,MAAA,CAAO,MAAA,GAAS,GAAA;AAAA,MAC1C;AAEA,MAAA,OAAO,KAAA;AAAA,IACT,CAAC;AAAA,GACH;AACA,EAAA,OAAO,OAAA,CAAQ,SAAS,QAAA,CAAS,MAAA;AACnC;AAEO,SAAS,wBAAA,GAA2B;AACzC,EAAA,OAAO,YAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GACJ,CAAC,GAAA,CAAI,KAAA,IACL,IAAI,KAAA,CAAM,aAAA,CAAc,IAAA,CAAK,CAAC,MAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAExG,IAAA,MAAM,eAAA,GACJ,CAAC,GAAA,CAAI,MAAA,IAAU,IAAI,MAAA,CAAO,IAAA,CAAK,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAE1G,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,oCAAoC,CAAA;AAAA,IACtD;AAEA,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAChG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,EAAE,OAAA,EAAQ,KAA2B,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE5F,IAAA,MAAM,cAAA,GAAiB,KAAA;AACvB,IAAA,MAAM,eAAA,GAAkB,MAAA;AAExB,IAAA,MAAM,QAAA,GAAW,GAAA,CAAI,cAAA,CAAe,IAAA,EAAM,CAAA;AAC1C,IAAA,MAAM,SAAA,GAAY,GAAA,CAAI,eAAA,CAAgB,IAAA,EAAM,CAAA;AAG5C,IAAA,MAAM,aAAA,GAAgB,gBAAgB,QAAQ,CAAA;AAC9C,IAAA,MAAM,cAAA,GAAiB,gBAAgB,SAAS,CAAA;AAEhD,IAAA,OAAO;AAAA,MACL,aAAA;AAAA,MACA,cAAA;AAAA,MACA,eAAA,EAAiB,cAAc,MAAA,CAAO,CAAA,CAAA,KAAK,CAAC,cAAA,CAAe,QAAA,CAAS,CAAC,CAAC,CAAA;AAAA,MACtE,aAAA,EAAe;AAAA,QACb,OAAO,aAAA,CAAc,MAAA;AAAA,QACrB,QAAQ,cAAA,CAAe;AAAA;AACzB,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAA,GAAgB,QAAQ,oBAAA,EAAsB,aAAA;AACpD,IAAA,MAAM,cAAA,GAAiB,QAAQ,oBAAA,EAAsB,cAAA;AAErD,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,QAAA,EAAU,aAAA;AAAA,MACV,UAAA,EAAY;AAAA,KACb,CAAA;AAAA,EACH,CAAC,CAAA;AACL;ACzHO,SAAS,6BAAA,GAAgC;AAC9C,EAAA,OAAOA,YAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACpF,IAAA,MAAM,OAAA,GAAU,IAAI,eAAA,CAAgB,IAAA,EAAM,OAAO,MAAM,CAAA;AACvD,IAAA,MAAM,KAAA,GAAQ,QAAQ,KAAA,EAAM;AAG5B,IAAA,MAAM,GAAA,GAAM,QAAQ,UAAA,EAAW;AAC/B,IAAA,MAAM,OAAA,GAAU,IAAI,MAAA,CAAO,CAAC,CAAC,EAAE,CAAA,KAAM,EAAA,KAAO,OAAO,CAAA,CAAE,MAAA;AAGrD,IAAA,MAAM,YAAY,IAAA,CAAK,GAAA,CAAI,KAAA,CAAM,MAAA,EAAQ,OAAO,MAAM,CAAA;AACtD,IAAA,MAAM,UAAA,GAAa,SAAA,GAAY,CAAA,GAAI,IAAA,CAAK,GAAA,CAAI,MAAM,MAAA,GAAS,MAAA,CAAO,MAAM,CAAA,GAAI,SAAA,GAAY,CAAA;AACxF,IAAA,MAAM,aAAa,CAAA,GAAI,UAAA;AAEvB,IAAA,OAAO;AAAA,MACL,KAAA;AAAA,MACA,UAAA;AAAA,MACA,OAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;AC/BO,SAAS,2BAAA,GAA8B;AAC5C,EAAA,OAAOA,YAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAEpF,IAAA,IAAI,CAAC,KAAA,IAAS,CAAC,MAAA,EAAQ;AACrB,MAAA,OAAO;AAAA,QACL,MAAA,EAAQ;AAAA,UACN,iBAAA,sBAAuB,GAAA,EAAY;AAAA,UACnC,gBAAA,sBAAsB,GAAA;AAAY;AACpC,OACF;AAAA,IACF;AAEA,IAAA,MAAM,eAAA,GAAkB,CAAC,IAAA,KAAiB;AACxC,MAAA,OAAO,iBAAA,CAAkB,QAAQ,IAAA,EAAM;AAAA,QACrC,QAAA,EAAU,SAAA;AAAA,QACV,aAAA,EAAe,IAAA;AAAA,QACf,mBAAA,EAAqB,IAAA;AAAA,QACrB,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAAA,IACH,CAAA;AAEA,IAAA,MAAM,iBAAA,GAAoB,IAAI,GAAA,CAAI,eAAA,CAAgB,KAAK,CAAC,CAAA;AACxD,IAAA,MAAM,gBAAA,GAAmB,IAAI,GAAA,CAAI,eAAA,CAAgB,MAAM,CAAC,CAAA;AACxD,IAAA,OAAO;AAAA,MACL,iBAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,OAAA,CAAQ,OAAO,EAAE,SAAQ,KAAM;AAC9B,IAAA,IACE,CAAC,QAAQ,oBAAA,EAAsB,iBAAA,EAAmB,QAClD,CAAC,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAA,EACjD;AACA,MAAA,OAAO;AAAA,QACL,mBAAA,EAAqB,CAAA;AAAA,QACrB,qBAAA,EAAuB;AAAA,OACzB;AAAA,IACF;AAEA,IAAA,MAAM,kBAAkB,CAAC,GAAG,OAAA,CAAQ,oBAAA,EAAsB,iBAAiB,CAAA,CAAE,MAAA;AAAA,MAAO,CAAA,CAAA,KAClF,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAI,CAAC;AAAA,KACvD;AAEA,IAAA,OAAO;AAAA,MACL,qBAAqB,KAAA,CAAM,IAAA,CAAK,QAAQ,oBAAA,EAAsB,iBAAiB,EAAE,MAAA,IAAU,CAAA;AAAA,MAC3F,qBAAA,EAAuB,gBAAgB,MAAA,IAAU;AAAA,KACnD;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,IAAI,CAAC,OAAA,CAAQ,iBAAA,EAAmB,mBAAA,EAAqB;AACnD,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,MAAM,aAAA,GAAgB,QAAQ,iBAAA,EAAmB,mBAAA;AACjD,IAAA,MAAM,eAAA,GAAkB,QAAQ,iBAAA,EAAmB,qBAAA;AACnD,IAAA,OAAO,aAAA,GAAgB,CAAA,GAAI,eAAA,GAAkB,aAAA,GAAgB,CAAA;AAAA,EAC/D,CAAC,CAAA;AACL;AC3DO,SAAS,6BAAA,CACd,EAAE,UAAA,EAAY,gBAAA,EAAiB,GAA8B,EAAE,UAAA,EAAY,IAAA,EAAM,gBAAA,EAAkB,IAAA,EAAK,EACxG;AACA,EAAA,OAAOA,YAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,IAAI,cAAA,GAAiB,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACvG,IAAA,IAAI,eAAA,GAAkB,GAAA,CAAI,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE1F,IAAA,IAAI,UAAA,EAAY;AACd,MAAA,cAAA,GAAiB,eAAe,WAAA,EAAY;AAC5C,MAAA,eAAA,GAAkB,gBAAgB,WAAA,EAAY;AAAA,IAChD;AAEA,IAAA,IAAI,gBAAA,EAAkB;AACpB,MAAA,cAAA,GAAiB,cAAA,CAAe,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAC1D,MAAA,eAAA,GAAkB,eAAA,CAAgB,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAAA,IAC9D;AAEA,IAAA,OAAO;AAAA,MACL,cAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAa,gBAAA,CAAiB,iBAAA;AAAA,MAClC,QAAQ,oBAAA,EAAsB,cAAA;AAAA,MAC9B,QAAQ,oBAAA,EAAsB;AAAA,KAChC;AAEA,IAAA,OAAO,UAAA;AAAA,EACT,CAAC,CAAA;AACL;ACpCO,SAAS,gBAAA,CAAiB,MAAA,GAA2B,EAAC,EAAG;AAC9D,EAAA,MAAM,EAAE,eAAc,GAAI,MAAA;AAE1B,EAAA,OAAOA,YAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,SAAA,GAAY,IAAI,SAAA,EAAU;AAChC,IAAA,MAAM,YAAA,GAAuB,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAClG,IAAA,MAAM,iBAAA,GAAoB,SAAA,CAAU,OAAA,CAAQ,YAAY,CAAA;AAExD,IAAA,IAAI,aAAA,EAAe;AAEjB,MAAA,MAAM,kBAAA,GAAqB,SAAA,CAAU,OAAA,CAAQ,aAAa,CAAA;AAC1D,MAAA,MAAM,gBAAgB,IAAA,CAAK,GAAA,CAAI,iBAAA,CAAkB,WAAA,GAAc,mBAAmB,WAAW,CAAA;AAC7F,MAAA,MAAM,eAAA,GAAkB,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,aAAa,CAAA;AAErD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,eAAA;AAAA,QACP,mBAAmB,iBAAA,CAAkB,WAAA;AAAA,QACrC,oBAAoB,kBAAA,CAAmB,WAAA;AAAA,QACvC,UAAA,EAAY;AAAA,OACd;AAAA,IACF;AAGA,IAAA,MAAM,YAAY,YAAA,CAAa,KAAA,CAAM,gBAAgB,CAAA,IAAK,CAAC,YAAY,CAAA;AACvE,IAAA,MAAM,UAAA,GAAa,UAAU,GAAA,CAAI,CAAA,CAAA,KAAK,UAAU,OAAA,CAAQ,CAAC,EAAE,WAAW,CAAA;AACtE,IAAA,MAAM,YAAA,GAAe,UAAA,CAAW,MAAA,CAAO,CAAC,CAAA,EAAG,MAAM,CAAA,GAAI,CAAA,EAAG,CAAC,CAAA,GAAI,UAAA,CAAW,MAAA;AACxE,IAAA,MAAM,QAAA,GAAW,UAAA,CAAW,MAAA,CAAO,CAAC,KAAK,CAAA,KAAM,GAAA,GAAM,IAAA,CAAK,GAAA,CAAI,IAAI,YAAA,EAAc,CAAC,CAAA,EAAG,CAAC,IAAI,UAAA,CAAW,MAAA;AACpG,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,QAAQ,CAAA;AAE1C,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,SAAA;AAAA,MACP,YAAA;AAAA,MACA,iBAAA,EAAmB;AAAA,KACrB;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL","file":"index.js","sourcesContent":["import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport nlp from 'compromise';\n\nfunction normalizeString(str: string): string {\n  // Remove diacritics and convert to lowercase\n  return str\n    .normalize('NFD')\n    .replace(/[\\u0300-\\u036f]/g, '')\n    .toLowerCase();\n}\n\nfunction extractElements(doc: any): string[] {\n  // Get more specific elements and ensure they're arrays\n  const nouns = doc.nouns().out('array') || [];\n  const verbs = doc.verbs().toInfinitive().out('array') || [];\n  const topics = doc.topics().out('array') || [];\n  const terms = doc.terms().out('array') || [];\n\n  // Helper function to clean and split terms\n  const cleanAndSplitTerm = (term: string): string[] => {\n    // First normalize the string\n    const normalized = normalizeString(term);\n\n    // Split on word boundaries and filter out empty strings\n    return normalized\n      .replace(/([a-z])([A-Z])/g, '$1 $2') // Split camelCase\n      .replace(/[^a-z0-9]+/g, ' ') // Replace non-alphanumeric with spaces\n      .trim()\n      .split(/\\s+/)\n      .filter(word => word.length > 0);\n  };\n\n  // Process all elements\n  const processedTerms = [\n    ...nouns.flatMap(cleanAndSplitTerm),\n    ...verbs.flatMap(cleanAndSplitTerm),\n    ...topics.flatMap(cleanAndSplitTerm),\n    ...terms.flatMap(cleanAndSplitTerm),\n  ];\n\n  // Remove duplicates\n  return [...new Set(processedTerms)];\n}\n\nfunction calculateCoverage({ original, simplified }: { original: string[]; simplified: string[] }): number {\n  if (original.length === 0) {\n    return simplified.length === 0 ? 1 : 0;\n  }\n\n  // Exact matching for short words (3 chars or less), substring matching for longer words\n  const covered = original.filter(element =>\n    simplified.some(s => {\n      const elem = normalizeString(element);\n      const simp = normalizeString(s);\n\n      // For short words (3 chars or less), require exact match\n      if (elem.length <= 3) {\n        return elem === simp;\n      }\n\n      // For longer words, require substantial overlap (more than 60% of the longer word)\n      const longer = elem.length > simp.length ? elem : simp;\n      const shorter = elem.length > simp.length ? simp : elem;\n\n      if (longer.includes(shorter)) {\n        return shorter.length / longer.length > 0.6;\n      }\n\n      return false;\n    }),\n  );\n  return covered.length / original.length;\n}\n\nexport function createCompletenessScorer() {\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      const isInputInvalid =\n        !run.input ||\n        run.input.inputMessages.some((i: { content: string }) => i.content === null || i.content === undefined);\n\n      const isOutputInvalid =\n        !run.output || run.output.some((i: { content: string }) => i.content === null || i.content === undefined);\n\n      if (isInputInvalid || isOutputInvalid) {\n        throw new Error('Inputs cannot be null or undefined');\n      }\n\n      const input = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map(({ content }: { content: string }) => content).join(', ') || '';\n\n      const inputToProcess = input;\n      const outputToProcess = output;\n\n      const inputDoc = nlp(inputToProcess.trim());\n      const outputDoc = nlp(outputToProcess.trim());\n\n      // Extract and log elements\n      const inputElements = extractElements(inputDoc);\n      const outputElements = extractElements(outputDoc);\n\n      return {\n        inputElements,\n        outputElements,\n        missingElements: inputElements.filter(e => !outputElements.includes(e)),\n        elementCounts: {\n          input: inputElements.length,\n          output: outputElements.length,\n        },\n      };\n    })\n    .generateScore(({ results }) => {\n      const inputElements = results.preprocessStepResult?.inputElements;\n      const outputElements = results.preprocessStepResult?.outputElements;\n\n      return calculateCoverage({\n        original: inputElements,\n        simplified: outputElements,\n      });\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport { SequenceMatcher } from 'difflib';\n\nexport function createTextualDifferenceScorer() {\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n      const matcher = new SequenceMatcher(null, input, output);\n      const ratio = matcher.ratio();\n\n      // Get detailed operations\n      const ops = matcher.getOpcodes();\n      const changes = ops.filter(([op]) => op !== 'equal').length;\n\n      // Calculate confidence based on text length difference\n      const maxLength = Math.max(input.length, output.length);\n      const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;\n      const confidence = 1 - lengthDiff;\n\n      return {\n        ratio,\n        confidence,\n        changes,\n        lengthDiff,\n      };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.ratio;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport keyword_extractor from 'keyword-extractor';\n\nexport function createKeywordCoverageScorer() {\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n\n      if (!input && !output) {\n        return {\n          result: {\n            referenceKeywords: new Set<string>(),\n            responseKeywords: new Set<string>(),\n          },\n        };\n      }\n\n      const extractKeywords = (text: string) => {\n        return keyword_extractor.extract(text, {\n          language: 'english',\n          remove_digits: true,\n          return_changed_case: true,\n          remove_duplicates: true,\n        });\n      };\n\n      const referenceKeywords = new Set(extractKeywords(input));\n      const responseKeywords = new Set(extractKeywords(output));\n      return {\n        referenceKeywords,\n        responseKeywords,\n      };\n    })\n    .analyze(async ({ results }) => {\n      if (\n        !results.preprocessStepResult?.referenceKeywords?.size &&\n        !results.preprocessStepResult?.responseKeywords?.size\n      ) {\n        return {\n          totalKeywordsLength: 0,\n          matchedKeywordsLength: 0,\n        };\n      }\n\n      const matchedKeywords = [...results.preprocessStepResult?.referenceKeywords].filter(k =>\n        results.preprocessStepResult?.responseKeywords?.has(k),\n      );\n\n      return {\n        totalKeywordsLength: Array.from(results.preprocessStepResult?.referenceKeywords).length ?? 0,\n        matchedKeywordsLength: matchedKeywords.length ?? 0,\n      };\n    })\n    .generateScore(({ results }) => {\n      if (!results.analyzeStepResult?.totalKeywordsLength) {\n        return 1;\n      }\n\n      const totalKeywords = results.analyzeStepResult?.totalKeywordsLength!;\n      const matchedKeywords = results.analyzeStepResult?.matchedKeywordsLength!;\n      return totalKeywords > 0 ? matchedKeywords / totalKeywords : 0;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport stringSimilarity from 'string-similarity';\n\ninterface ContentSimilarityOptions {\n  ignoreCase?: boolean;\n  ignoreWhitespace?: boolean;\n}\n\nexport function createContentSimilarityScorer(\n  { ignoreCase, ignoreWhitespace }: ContentSimilarityOptions = { ignoreCase: true, ignoreWhitespace: true },\n) {\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      let processedInput = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') || '';\n      let processedOutput = run.output.map((i: { content: string }) => i.content).join(', ') || '';\n\n      if (ignoreCase) {\n        processedInput = processedInput.toLowerCase();\n        processedOutput = processedOutput.toLowerCase();\n      }\n\n      if (ignoreWhitespace) {\n        processedInput = processedInput.replace(/\\s+/g, ' ').trim();\n        processedOutput = processedOutput.replace(/\\s+/g, ' ').trim();\n      }\n\n      return {\n        processedInput,\n        processedOutput,\n      };\n    })\n    .generateScore(({ results }) => {\n      const similarity = stringSimilarity.compareTwoStrings(\n        results.preprocessStepResult?.processedInput,\n        results.preprocessStepResult?.processedOutput,\n      );\n\n      return similarity;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport Sentiment from 'sentiment';\n\ninterface ToneScorerConfig {\n  referenceTone?: string;\n}\n\nexport function createToneScorer(config: ToneScorerConfig = {}) {\n  const { referenceTone } = config;\n\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      const sentiment = new Sentiment();\n      const agentMessage: string = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n      const responseSentiment = sentiment.analyze(agentMessage);\n\n      if (referenceTone) {\n        // Compare sentiment with reference\n        const referenceSentiment = sentiment.analyze(referenceTone);\n        const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);\n        const normalizedScore = Math.max(0, 1 - sentimentDiff);\n\n        return {\n          score: normalizedScore,\n          responseSentiment: responseSentiment.comparative,\n          referenceSentiment: referenceSentiment.comparative,\n          difference: sentimentDiff,\n        };\n      }\n\n      // Evaluate sentiment stability across response\n      const sentences = agentMessage.match(/[^.!?]+[.!?]+/g) || [agentMessage];\n      const sentiments = sentences.map(s => sentiment.analyze(s).comparative);\n      const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;\n      const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;\n      const stability = Math.max(0, 1 - variance);\n\n      return {\n        score: stability,\n        avgSentiment,\n        sentimentVariance: variance,\n      };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.score;\n    });\n}\n"]}
+{"version":3,"sources":["../../../src/scorers/code/completeness/index.ts","../../../src/scorers/code/textual-difference/index.ts","../../../src/scorers/code/keyword-coverage/index.ts","../../../src/scorers/code/content-similarity/index.ts","../../../src/scorers/code/tone/index.ts","../../../src/scorers/code/tool-call-accuracy/index.ts"],"names":["createScorer"],"mappings":";;;;;;;;AAIA,SAAS,gBAAgB,GAAA,EAAqB;AAE5C,EAAA,OAAO,GAAA,CACJ,UAAU,KAAK,CAAA,CACf,QAAQ,kBAAA,EAAoB,EAAE,EAC9B,WAAA,EAAY;AACjB;AAEA,SAAS,gBAAgB,GAAA,EAAoB;AAE3C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAC3C,EAAA,MAAM,KAAA,GAAQ,IAAI,KAAA,EAAM,CAAE,cAAa,CAAE,GAAA,CAAI,OAAO,CAAA,IAAK,EAAC;AAC1D,EAAA,MAAM,SAAS,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,OAAO,KAAK,EAAC;AAC7C,EAAA,MAAM,QAAQ,GAAA,CAAI,KAAA,GAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAG3C,EAAA,MAAM,iBAAA,GAAoB,CAAC,IAAA,KAA2B;AAEpD,IAAA,MAAM,UAAA,GAAa,gBAAgB,IAAI,CAAA;AAGvC,IAAA,OAAO,WACJ,OAAA,CAAQ,iBAAA,EAAmB,OAAO,CAAA,CAClC,OAAA,CAAQ,eAAe,GAAG,CAAA,CAC1B,IAAA,EAAK,CACL,MAAM,KAAK,CAAA,CACX,OAAO,CAAA,IAAA,KAAQ,IAAA,CAAK,SAAS,CAAC,CAAA;AAAA,EACnC,CAAA;AAGA,EAAA,MAAM,cAAA,GAAiB;AAAA,IACrB,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IAClC,GAAG,MAAA,CAAO,OAAA,CAAQ,iBAAiB,CAAA;AAAA,IACnC,GAAG,KAAA,CAAM,OAAA,CAAQ,iBAAiB;AAAA,GACpC;AAGA,EAAA,OAAO,CAAC,GAAG,IAAI,GAAA,CAAI,cAAc,CAAC,CAAA;AACpC;AAEA,SAAS,iBAAA,CAAkB,EAAE,QAAA,EAAU,UAAA,EAAW,EAAyD;AACzG,EAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,IAAA,OAAO,UAAA,CAAW,MAAA,KAAW,CAAA,GAAI,CAAA,GAAI,CAAA;AAAA,EACvC;AAGA,EAAA,MAAM,UAAU,QAAA,CAAS,MAAA;AAAA,IAAO,CAAA,OAAA,KAC9B,UAAA,CAAW,IAAA,CAAK,CAAA,CAAA,KAAK;AACnB,MAAA,MAAM,IAAA,GAAO,gBAAgB,OAAO,CAAA;AACpC,MAAA,MAAM,IAAA,GAAO,gBAAgB,CAAC,CAAA;AAG9B,MAAA,IAAI,IAAA,CAAK,UAAU,CAAA,EAAG;AACpB,QAAA,OAAO,IAAA,KAAS,IAAA;AAAA,MAClB;AAGA,MAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAClD,MAAA,MAAM,OAAA,GAAU,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,SAAS,IAAA,GAAO,IAAA;AAEnD,MAAA,IAAI,MAAA,CAAO,QAAA,CAAS,OAAO,CAAA,EAAG;AAC5B,QAAA,OAAO,OAAA,CAAQ,MAAA,GAAS,MAAA,CAAO,MAAA,GAAS,GAAA;AAAA,MAC1C;AAEA,MAAA,OAAO,KAAA;AAAA,IACT,CAAC;AAAA,GACH;AACA,EAAA,OAAO,OAAA,CAAQ,SAAS,QAAA,CAAS,MAAA;AACnC;AAEO,SAAS,wBAAA,GAA2B;AACzC,EAAA,OAAO,YAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GACJ,CAAC,GAAA,CAAI,KAAA,IACL,IAAI,KAAA,CAAM,aAAA,CAAc,IAAA,CAAK,CAAC,MAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAExG,IAAA,MAAM,eAAA,GACJ,CAAC,GAAA,CAAI,MAAA,IAAU,IAAI,MAAA,CAAO,IAAA,CAAK,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAA,KAAY,IAAA,IAAQ,CAAA,CAAE,YAAY,MAAS,CAAA;AAE1G,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,oCAAoC,CAAA;AAAA,IACtD;AAEA,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAChG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,EAAE,OAAA,EAAQ,KAA2B,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE5F,IAAA,MAAM,cAAA,GAAiB,KAAA;AACvB,IAAA,MAAM,eAAA,GAAkB,MAAA;AAExB,IAAA,MAAM,QAAA,GAAW,GAAA,CAAI,cAAA,CAAe,IAAA,EAAM,CAAA;AAC1C,IAAA,MAAM,SAAA,GAAY,GAAA,CAAI,eAAA,CAAgB,IAAA,EAAM,CAAA;AAG5C,IAAA,MAAM,aAAA,GAAgB,gBAAgB,QAAQ,CAAA;AAC9C,IAAA,MAAM,cAAA,GAAiB,gBAAgB,SAAS,CAAA;AAEhD,IAAA,OAAO;AAAA,MACL,aAAA;AAAA,MACA,cAAA;AAAA,MACA,eAAA,EAAiB,cAAc,MAAA,CAAO,CAAA,CAAA,KAAK,CAAC,cAAA,CAAe,QAAA,CAAS,CAAC,CAAC,CAAA;AAAA,MACtE,aAAA,EAAe;AAAA,QACb,OAAO,aAAA,CAAc,MAAA;AAAA,QACrB,QAAQ,cAAA,CAAe;AAAA;AACzB,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAA,GAAgB,QAAQ,oBAAA,EAAsB,aAAA;AACpD,IAAA,MAAM,cAAA,GAAiB,QAAQ,oBAAA,EAAsB,cAAA;AAErD,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,QAAA,EAAU,aAAA;AAAA,MACV,UAAA,EAAY;AAAA,KACb,CAAA;AAAA,EACH,CAAC,CAAA;AACL;ACzHO,SAAS,6BAAA,GAAgC;AAC9C,EAAA,OAAOA,YAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACpF,IAAA,MAAM,OAAA,GAAU,IAAI,eAAA,CAAgB,IAAA,EAAM,OAAO,MAAM,CAAA;AACvD,IAAA,MAAM,KAAA,GAAQ,QAAQ,KAAA,EAAM;AAG5B,IAAA,MAAM,GAAA,GAAM,QAAQ,UAAA,EAAW;AAC/B,IAAA,MAAM,OAAA,GAAU,IAAI,MAAA,CAAO,CAAC,CAAC,EAAE,CAAA,KAAM,EAAA,KAAO,OAAO,CAAA,CAAE,MAAA;AAGrD,IAAA,MAAM,YAAY,IAAA,CAAK,GAAA,CAAI,KAAA,CAAM,MAAA,EAAQ,OAAO,MAAM,CAAA;AACtD,IAAA,MAAM,UAAA,GAAa,SAAA,GAAY,CAAA,GAAI,IAAA,CAAK,GAAA,CAAI,MAAM,MAAA,GAAS,MAAA,CAAO,MAAM,CAAA,GAAI,SAAA,GAAY,CAAA;AACxF,IAAA,MAAM,aAAa,CAAA,GAAI,UAAA;AAEvB,IAAA,OAAO;AAAA,MACL,KAAA;AAAA,MACA,UAAA;AAAA,MACA,OAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;AC/BO,SAAS,2BAAA,GAA8B;AAC5C,EAAA,OAAOA,YAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,KAAA,GAAQ,GAAA,CAAI,KAAA,EAAO,aAAA,EAAe,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACjG,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAEpF,IAAA,IAAI,CAAC,KAAA,IAAS,CAAC,MAAA,EAAQ;AACrB,MAAA,OAAO;AAAA,QACL,MAAA,EAAQ;AAAA,UACN,iBAAA,sBAAuB,GAAA,EAAY;AAAA,UACnC,gBAAA,sBAAsB,GAAA;AAAY;AACpC,OACF;AAAA,IACF;AAEA,IAAA,MAAM,eAAA,GAAkB,CAAC,IAAA,KAAiB;AACxC,MAAA,OAAO,iBAAA,CAAkB,QAAQ,IAAA,EAAM;AAAA,QACrC,QAAA,EAAU,SAAA;AAAA,QACV,aAAA,EAAe,IAAA;AAAA,QACf,mBAAA,EAAqB,IAAA;AAAA,QACrB,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAAA,IACH,CAAA;AAEA,IAAA,MAAM,iBAAA,GAAoB,IAAI,GAAA,CAAI,eAAA,CAAgB,KAAK,CAAC,CAAA;AACxD,IAAA,MAAM,gBAAA,GAAmB,IAAI,GAAA,CAAI,eAAA,CAAgB,MAAM,CAAC,CAAA;AACxD,IAAA,OAAO;AAAA,MACL,iBAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,OAAA,CAAQ,OAAO,EAAE,SAAQ,KAAM;AAC9B,IAAA,IACE,CAAC,QAAQ,oBAAA,EAAsB,iBAAA,EAAmB,QAClD,CAAC,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAA,EACjD;AACA,MAAA,OAAO;AAAA,QACL,mBAAA,EAAqB,CAAA;AAAA,QACrB,qBAAA,EAAuB;AAAA,OACzB;AAAA,IACF;AAEA,IAAA,MAAM,kBAAkB,CAAC,GAAG,OAAA,CAAQ,oBAAA,EAAsB,iBAAiB,CAAA,CAAE,MAAA;AAAA,MAAO,CAAA,CAAA,KAClF,OAAA,CAAQ,oBAAA,EAAsB,gBAAA,EAAkB,IAAI,CAAC;AAAA,KACvD;AAEA,IAAA,OAAO;AAAA,MACL,qBAAqB,KAAA,CAAM,IAAA,CAAK,QAAQ,oBAAA,EAAsB,iBAAiB,EAAE,MAAA,IAAU,CAAA;AAAA,MAC3F,qBAAA,EAAuB,gBAAgB,MAAA,IAAU;AAAA,KACnD;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,IAAI,CAAC,OAAA,CAAQ,iBAAA,EAAmB,mBAAA,EAAqB;AACnD,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,MAAM,aAAA,GAAgB,QAAQ,iBAAA,EAAmB,mBAAA;AACjD,IAAA,MAAM,eAAA,GAAkB,QAAQ,iBAAA,EAAmB,qBAAA;AACnD,IAAA,OAAO,aAAA,GAAgB,CAAA,GAAI,eAAA,GAAkB,aAAA,GAAgB,CAAA;AAAA,EAC/D,CAAC,CAAA;AACL;AC3DO,SAAS,6BAAA,CACd,EAAE,UAAA,EAAY,gBAAA,EAAiB,GAA8B,EAAE,UAAA,EAAY,IAAA,EAAM,gBAAA,EAAkB,IAAA,EAAK,EACxG;AACA,EAAA,OAAOA,YAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,IAAI,cAAA,GAAiB,GAAA,CAAI,KAAA,EAAO,aAAA,CAAc,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AACvG,IAAA,IAAI,eAAA,GAAkB,GAAA,CAAI,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAE1F,IAAA,IAAI,UAAA,EAAY;AACd,MAAA,cAAA,GAAiB,eAAe,WAAA,EAAY;AAC5C,MAAA,eAAA,GAAkB,gBAAgB,WAAA,EAAY;AAAA,IAChD;AAEA,IAAA,IAAI,gBAAA,EAAkB;AACpB,MAAA,cAAA,GAAiB,cAAA,CAAe,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAC1D,MAAA,eAAA,GAAkB,eAAA,CAAgB,OAAA,CAAQ,MAAA,EAAQ,GAAG,EAAE,IAAA,EAAK;AAAA,IAC9D;AAEA,IAAA,OAAO;AAAA,MACL,cAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,aAAa,gBAAA,CAAiB,iBAAA;AAAA,MAClC,QAAQ,oBAAA,EAAsB,cAAA;AAAA,MAC9B,QAAQ,oBAAA,EAAsB;AAAA,KAChC;AAEA,IAAA,OAAO,UAAA;AAAA,EACT,CAAC,CAAA;AACL;ACpCO,SAAS,gBAAA,CAAiB,MAAA,GAA2B,EAAC,EAAG;AAC9D,EAAA,MAAM,EAAE,eAAc,GAAI,MAAA;AAE1B,EAAA,OAAOA,YAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,cAAA;AAAA,IACN,WAAA,EACE;AAAA,GACH,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,SAAA,GAAY,IAAI,SAAA,EAAU;AAChC,IAAA,MAAM,YAAA,GAAuB,GAAA,CAAI,MAAA,EAAQ,GAAA,CAAI,CAAC,CAAA,KAA2B,CAAA,CAAE,OAAO,CAAA,CAAE,IAAA,CAAK,IAAI,CAAA,IAAK,EAAA;AAClG,IAAA,MAAM,iBAAA,GAAoB,SAAA,CAAU,OAAA,CAAQ,YAAY,CAAA;AAExD,IAAA,IAAI,aAAA,EAAe;AAEjB,MAAA,MAAM,kBAAA,GAAqB,SAAA,CAAU,OAAA,CAAQ,aAAa,CAAA;AAC1D,MAAA,MAAM,gBAAgB,IAAA,CAAK,GAAA,CAAI,iBAAA,CAAkB,WAAA,GAAc,mBAAmB,WAAW,CAAA;AAC7F,MAAA,MAAM,eAAA,GAAkB,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,aAAa,CAAA;AAErD,MAAA,OAAO;AAAA,QACL,KAAA,EAAO,eAAA;AAAA,QACP,mBAAmB,iBAAA,CAAkB,WAAA;AAAA,QACrC,oBAAoB,kBAAA,CAAmB,WAAA;AAAA,QACvC,UAAA,EAAY;AAAA,OACd;AAAA,IACF;AAGA,IAAA,MAAM,YAAY,YAAA,CAAa,KAAA,CAAM,gBAAgB,CAAA,IAAK,CAAC,YAAY,CAAA;AACvE,IAAA,MAAM,UAAA,GAAa,UAAU,GAAA,CAAI,CAAA,CAAA,KAAK,UAAU,OAAA,CAAQ,CAAC,EAAE,WAAW,CAAA;AACtE,IAAA,MAAM,YAAA,GAAe,UAAA,CAAW,MAAA,CAAO,CAAC,CAAA,EAAG,MAAM,CAAA,GAAI,CAAA,EAAG,CAAC,CAAA,GAAI,UAAA,CAAW,MAAA;AACxE,IAAA,MAAM,QAAA,GAAW,UAAA,CAAW,MAAA,CAAO,CAAC,KAAK,CAAA,KAAM,GAAA,GAAM,IAAA,CAAK,GAAA,CAAI,IAAI,YAAA,EAAc,CAAC,CAAA,EAAG,CAAC,IAAI,UAAA,CAAW,MAAA;AACpG,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,IAAI,QAAQ,CAAA;AAE1C,IAAA,OAAO;AAAA,MACL,KAAA,EAAO,SAAA;AAAA,MACP,YAAA;AAAA,MACA,iBAAA,EAAmB;AAAA,KACrB;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,OAAO,QAAQ,oBAAA,EAAsB,KAAA;AAAA,EACvC,CAAC,CAAA;AACL;ACzCA,SAAS,cAAA,CAAe,WAAA,EAAuB,aAAA,EAAyB,UAAA,GAAsB,KAAA,EAAgB;AAC5G,EAAA,IAAI,UAAA,EAAY;AACd,IAAA,OAAO,KAAK,SAAA,CAAU,WAAW,CAAA,KAAM,IAAA,CAAK,UAAU,aAAa,CAAA;AAAA,EACrE;AAEA,EAAA,MAAM,kBAA4B,EAAC;AACnC,EAAA,KAAA,MAAW,gBAAgB,aAAA,EAAe;AACxC,IAAA,MAAM,KAAA,GAAQ,WAAA,CAAY,OAAA,CAAQ,YAAY,CAAA;AAC9C,IAAA,IAAI,UAAU,EAAA,EAAI;AAChB,MAAA,OAAO,KAAA;AAAA,IACT;AACA,IAAA,eAAA,CAAgB,KAAK,KAAK,CAAA;AAAA,EAC5B;AAEA,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,eAAA,CAAgB,QAAQ,CAAA,EAAA,EAAK;AAC/C,IAAA,MAAM,YAAA,GAAe,gBAAgB,CAAC,CAAA;AACtC,IAAA,MAAM,SAAA,GAAY,eAAA,CAAgB,CAAA,GAAI,CAAC,CAAA;AACvC,IAAA,IAAI,YAAA,KAAiB,MAAA,IAAa,SAAA,KAAc,MAAA,IAAa,gBAAgB,SAAA,EAAW;AACtF,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AAEA,EAAA,OAAO,IAAA;AACT;AAEA,SAAS,iBAAA,CAAkB;AAAA,EACzB,YAAA;AAAA,EACA,WAAA;AAAA,EACA,UAAA,GAAa,KAAA;AAAA,EACb;AACF,CAAA,EAKW;AACT,EAAA,IAAI,WAAA,CAAY,WAAW,CAAA,EAAG;AAC5B,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,IAAI,iBAAA,IAAqB,iBAAA,CAAkB,MAAA,GAAS,CAAA,EAAG;AACrD,IAAA,OAAO,cAAA,CAAe,WAAA,EAAa,iBAAA,EAAmB,UAAU,IAAI,CAAA,GAAI,CAAA;AAAA,EAC1E;AAEA,EAAA,IAAI,CAAC,YAAA,EAAc;AACjB,IAAA,OAAO,CAAA;AAAA,EACT;AAEA,EAAA,IAAI,UAAA,EAAY;AACd,IAAA,OAAO,YAAY,MAAA,KAAW,CAAA,IAAK,YAAY,CAAC,CAAA,KAAM,eAAe,CAAA,GAAI,CAAA;AAAA,EAC3E;AAEA,EAAA,OAAO,WAAA,CAAY,QAAA,CAAS,YAAY,CAAA,GAAI,CAAA,GAAI,CAAA;AAClD;AAEO,SAAS,iCAAiC,OAAA,EAAkC;AACjF,EAAA,MAAM,EAAE,YAAA,EAAc,UAAA,GAAa,KAAA,EAAO,mBAAkB,GAAI,OAAA;AAEhE,EAAA,IAAI,CAAC,YAAA,IAAgB,CAAC,iBAAA,EAAmB;AACvC,IAAA,MAAM,IAAI,MAAM,2DAA2D,CAAA;AAAA,EAC7E;AAEA,EAAA,MAAM,iBAAiB,MAAM;AAC3B,IAAA,OAAO,iBAAA,GACH,iEAAiE,iBAAA,CAAkB,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA,CAAA,GAC7F,wDAAwD,YAAY,CAAA,0BAAA,CAAA;AAAA,EAC1E,CAAA;AAEA,EAAA,OAAOA,YAAAA,CAA8D;AAAA,IACnE,IAAA,EAAM,oBAAA;AAAA,IACN,aAAa,cAAA;AAAe,GAC7B,CAAA,CACE,UAAA,CAAW,OAAO,EAAE,KAAI,KAAM;AAC7B,IAAA,MAAM,cAAA,GAAiB,CAAC,GAAA,CAAI,KAAA,IAAS,CAAC,GAAA,CAAI,KAAA,CAAM,aAAA,IAAiB,GAAA,CAAI,KAAA,CAAM,aAAA,CAAc,MAAA,KAAW,CAAA;AACpG,IAAA,MAAM,kBAAkB,CAAC,GAAA,CAAI,MAAA,IAAU,GAAA,CAAI,OAAO,MAAA,KAAW,CAAA;AAE7D,IAAA,IAAI,kBAAkB,eAAA,EAAiB;AACrC,MAAA,MAAM,IAAI,MAAM,mDAAmD,CAAA;AAAA,IACrE;AAEA,IAAA,MAAM,EAAE,KAAA,EAAO,WAAA,EAAa,eAAc,GAAI,gBAAA,CAAiB,IAAI,MAAM,CAAA;AAEzE,IAAA,MAAM,iBAAA,GAAoB,YAAA,GACtB,UAAA,GACE,WAAA,CAAY,MAAA,KAAW,CAAA,IAAK,WAAA,CAAY,CAAC,CAAA,KAAM,YAAA,GAC/C,WAAA,CAAY,QAAA,CAAS,YAAY,CAAA,GACnC,KAAA;AAEJ,IAAA,OAAO;AAAA,MACL,YAAA;AAAA,MACA,WAAA;AAAA,MACA,UAAA;AAAA,MACA,iBAAA;AAAA,MACA,YAAA,EAAc,YAAY,MAAA,GAAS,CAAA;AAAA,MACnC,iBAAA;AAAA,MACA,aAAA;AAAA,MACA,oBAAoB,iBAAA,GAAoB,cAAA,CAAe,WAAA,EAAa,iBAAA,EAAmB,UAAU,CAAA,GAAI;AAAA,KACvG;AAAA,EACF,CAAC,CAAA,CACA,aAAA,CAAc,CAAC,EAAE,SAAQ,KAAM;AAC9B,IAAA,MAAM,mBAAmB,OAAA,CAAQ,oBAAA;AACjC,IAAA,IAAI,CAAC,gBAAA,EAAkB;AACrB,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,OAAO,iBAAA,CAAkB;AAAA,MACvB,cAAc,gBAAA,CAAiB,YAAA;AAAA,MAC/B,aAAa,gBAAA,CAAiB,WAAA;AAAA,MAC9B,YAAY,gBAAA,CAAiB,UAAA;AAAA,MAC7B,mBAAmB,gBAAA,CAAiB;AAAA,KACrC,CAAA;AAAA,EACH,CAAC,CAAA;AACL","file":"index.js","sourcesContent":["import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport nlp from 'compromise';\n\nfunction normalizeString(str: string): string {\n  // Remove diacritics and convert to lowercase\n  return str\n    .normalize('NFD')\n    .replace(/[\\u0300-\\u036f]/g, '')\n    .toLowerCase();\n}\n\nfunction extractElements(doc: any): string[] {\n  // Get more specific elements and ensure they're arrays\n  const nouns = doc.nouns().out('array') || [];\n  const verbs = doc.verbs().toInfinitive().out('array') || [];\n  const topics = doc.topics().out('array') || [];\n  const terms = doc.terms().out('array') || [];\n\n  // Helper function to clean and split terms\n  const cleanAndSplitTerm = (term: string): string[] => {\n    // First normalize the string\n    const normalized = normalizeString(term);\n\n    // Split on word boundaries and filter out empty strings\n    return normalized\n      .replace(/([a-z])([A-Z])/g, '$1 $2') // Split camelCase\n      .replace(/[^a-z0-9]+/g, ' ') // Replace non-alphanumeric with spaces\n      .trim()\n      .split(/\\s+/)\n      .filter(word => word.length > 0);\n  };\n\n  // Process all elements\n  const processedTerms = [\n    ...nouns.flatMap(cleanAndSplitTerm),\n    ...verbs.flatMap(cleanAndSplitTerm),\n    ...topics.flatMap(cleanAndSplitTerm),\n    ...terms.flatMap(cleanAndSplitTerm),\n  ];\n\n  // Remove duplicates\n  return [...new Set(processedTerms)];\n}\n\nfunction calculateCoverage({ original, simplified }: { original: string[]; simplified: string[] }): number {\n  if (original.length === 0) {\n    return simplified.length === 0 ? 1 : 0;\n  }\n\n  // Exact matching for short words (3 chars or less), substring matching for longer words\n  const covered = original.filter(element =>\n    simplified.some(s => {\n      const elem = normalizeString(element);\n      const simp = normalizeString(s);\n\n      // For short words (3 chars or less), require exact match\n      if (elem.length <= 3) {\n        return elem === simp;\n      }\n\n      // For longer words, require substantial overlap (more than 60% of the longer word)\n      const longer = elem.length > simp.length ? elem : simp;\n      const shorter = elem.length > simp.length ? simp : elem;\n\n      if (longer.includes(shorter)) {\n        return shorter.length / longer.length > 0.6;\n      }\n\n      return false;\n    }),\n  );\n  return covered.length / original.length;\n}\n\nexport function createCompletenessScorer() {\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      const isInputInvalid =\n        !run.input ||\n        run.input.inputMessages.some((i: { content: string }) => i.content === null || i.content === undefined);\n\n      const isOutputInvalid =\n        !run.output || run.output.some((i: { content: string }) => i.content === null || i.content === undefined);\n\n      if (isInputInvalid || isOutputInvalid) {\n        throw new Error('Inputs cannot be null or undefined');\n      }\n\n      const input = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map(({ content }: { content: string }) => content).join(', ') || '';\n\n      const inputToProcess = input;\n      const outputToProcess = output;\n\n      const inputDoc = nlp(inputToProcess.trim());\n      const outputDoc = nlp(outputToProcess.trim());\n\n      // Extract and log elements\n      const inputElements = extractElements(inputDoc);\n      const outputElements = extractElements(outputDoc);\n\n      return {\n        inputElements,\n        outputElements,\n        missingElements: inputElements.filter(e => !outputElements.includes(e)),\n        elementCounts: {\n          input: inputElements.length,\n          output: outputElements.length,\n        },\n      };\n    })\n    .generateScore(({ results }) => {\n      const inputElements = results.preprocessStepResult?.inputElements;\n      const outputElements = results.preprocessStepResult?.outputElements;\n\n      return calculateCoverage({\n        original: inputElements,\n        simplified: outputElements,\n      });\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport { SequenceMatcher } from 'difflib';\n\nexport function createTextualDifferenceScorer() {\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n      const matcher = new SequenceMatcher(null, input, output);\n      const ratio = matcher.ratio();\n\n      // Get detailed operations\n      const ops = matcher.getOpcodes();\n      const changes = ops.filter(([op]) => op !== 'equal').length;\n\n      // Calculate confidence based on text length difference\n      const maxLength = Math.max(input.length, output.length);\n      const lengthDiff = maxLength > 0 ? Math.abs(input.length - output.length) / maxLength : 0;\n      const confidence = 1 - lengthDiff;\n\n      return {\n        ratio,\n        confidence,\n        changes,\n        lengthDiff,\n      };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.ratio;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport keyword_extractor from 'keyword-extractor';\n\nexport function createKeywordCoverageScorer() {\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      const input = run.input?.inputMessages?.map((i: { content: string }) => i.content).join(', ') || '';\n      const output = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n\n      if (!input && !output) {\n        return {\n          result: {\n            referenceKeywords: new Set<string>(),\n            responseKeywords: new Set<string>(),\n          },\n        };\n      }\n\n      const extractKeywords = (text: string) => {\n        return keyword_extractor.extract(text, {\n          language: 'english',\n          remove_digits: true,\n          return_changed_case: true,\n          remove_duplicates: true,\n        });\n      };\n\n      const referenceKeywords = new Set(extractKeywords(input));\n      const responseKeywords = new Set(extractKeywords(output));\n      return {\n        referenceKeywords,\n        responseKeywords,\n      };\n    })\n    .analyze(async ({ results }) => {\n      if (\n        !results.preprocessStepResult?.referenceKeywords?.size &&\n        !results.preprocessStepResult?.responseKeywords?.size\n      ) {\n        return {\n          totalKeywordsLength: 0,\n          matchedKeywordsLength: 0,\n        };\n      }\n\n      const matchedKeywords = [...results.preprocessStepResult?.referenceKeywords].filter(k =>\n        results.preprocessStepResult?.responseKeywords?.has(k),\n      );\n\n      return {\n        totalKeywordsLength: Array.from(results.preprocessStepResult?.referenceKeywords).length ?? 0,\n        matchedKeywordsLength: matchedKeywords.length ?? 0,\n      };\n    })\n    .generateScore(({ results }) => {\n      if (!results.analyzeStepResult?.totalKeywordsLength) {\n        return 1;\n      }\n\n      const totalKeywords = results.analyzeStepResult?.totalKeywordsLength!;\n      const matchedKeywords = results.analyzeStepResult?.matchedKeywordsLength!;\n      return totalKeywords > 0 ? matchedKeywords / totalKeywords : 0;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport stringSimilarity from 'string-similarity';\n\ninterface ContentSimilarityOptions {\n  ignoreCase?: boolean;\n  ignoreWhitespace?: boolean;\n}\n\nexport function createContentSimilarityScorer(\n  { ignoreCase, ignoreWhitespace }: ContentSimilarityOptions = { ignoreCase: true, ignoreWhitespace: true },\n) {\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      let processedInput = run.input?.inputMessages.map((i: { content: string }) => i.content).join(', ') || '';\n      let processedOutput = run.output.map((i: { content: string }) => i.content).join(', ') || '';\n\n      if (ignoreCase) {\n        processedInput = processedInput.toLowerCase();\n        processedOutput = processedOutput.toLowerCase();\n      }\n\n      if (ignoreWhitespace) {\n        processedInput = processedInput.replace(/\\s+/g, ' ').trim();\n        processedOutput = processedOutput.replace(/\\s+/g, ' ').trim();\n      }\n\n      return {\n        processedInput,\n        processedOutput,\n      };\n    })\n    .generateScore(({ results }) => {\n      const similarity = stringSimilarity.compareTwoStrings(\n        results.preprocessStepResult?.processedInput,\n        results.preprocessStepResult?.processedOutput,\n      );\n\n      return similarity;\n    });\n}\n","import { createScorer } from '@mastra/core/scores';\nimport type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport Sentiment from 'sentiment';\n\ninterface ToneScorerConfig {\n  referenceTone?: string;\n}\n\nexport function createToneScorer(config: ToneScorerConfig = {}) {\n  const { referenceTone } = config;\n\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Completeness',\n    description:\n      'Leverage the nlp method from \"compromise\" to extract elements from the input and output and calculate the coverage.',\n  })\n    .preprocess(async ({ run }) => {\n      const sentiment = new Sentiment();\n      const agentMessage: string = run.output?.map((i: { content: string }) => i.content).join(', ') || '';\n      const responseSentiment = sentiment.analyze(agentMessage);\n\n      if (referenceTone) {\n        // Compare sentiment with reference\n        const referenceSentiment = sentiment.analyze(referenceTone);\n        const sentimentDiff = Math.abs(responseSentiment.comparative - referenceSentiment.comparative);\n        const normalizedScore = Math.max(0, 1 - sentimentDiff);\n\n        return {\n          score: normalizedScore,\n          responseSentiment: responseSentiment.comparative,\n          referenceSentiment: referenceSentiment.comparative,\n          difference: sentimentDiff,\n        };\n      }\n\n      // Evaluate sentiment stability across response\n      const sentences = agentMessage.match(/[^.!?]+[.!?]+/g) || [agentMessage];\n      const sentiments = sentences.map(s => sentiment.analyze(s).comparative);\n      const avgSentiment = sentiments.reduce((a, b) => a + b, 0) / sentiments.length;\n      const variance = sentiments.reduce((sum, s) => sum + Math.pow(s - avgSentiment, 2), 0) / sentiments.length;\n      const stability = Math.max(0, 1 - variance);\n\n      return {\n        score: stability,\n        avgSentiment,\n        sentimentVariance: variance,\n      };\n    })\n    .generateScore(({ results }) => {\n      return results.preprocessStepResult?.score;\n    });\n}\n","import type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';\nimport { createScorer } from '@mastra/core/scores';\nimport { extractToolCalls } from '../../utils';\n\ninterface ToolCallAccuracyOptions {\n  expectedTool?: string;\n  strictMode?: boolean;\n  expectedToolOrder?: string[];\n}\n\nfunction checkToolOrder(actualTools: string[], expectedOrder: string[], strictMode: boolean = false): boolean {\n  if (strictMode) {\n    return JSON.stringify(actualTools) === JSON.stringify(expectedOrder);\n  }\n\n  const expectedIndices: number[] = [];\n  for (const expectedTool of expectedOrder) {\n    const index = actualTools.indexOf(expectedTool);\n    if (index === -1) {\n      return false;\n    }\n    expectedIndices.push(index);\n  }\n\n  for (let i = 1; i < expectedIndices.length; i++) {\n    const currentIndex = expectedIndices[i];\n    const prevIndex = expectedIndices[i - 1];\n    if (currentIndex !== undefined && prevIndex !== undefined && currentIndex <= prevIndex) {\n      return false;\n    }\n  }\n\n  return true;\n}\n\nfunction calculateAccuracy({\n  expectedTool,\n  actualTools,\n  strictMode = false,\n  expectedToolOrder,\n}: {\n  expectedTool?: string;\n  actualTools: string[];\n  strictMode?: boolean;\n  expectedToolOrder?: string[];\n}): number {\n  if (actualTools.length === 0) {\n    return 0;\n  }\n\n  if (expectedToolOrder && expectedToolOrder.length > 0) {\n    return checkToolOrder(actualTools, expectedToolOrder, strictMode) ? 1 : 0;\n  }\n\n  if (!expectedTool) {\n    return 0;\n  }\n\n  if (strictMode) {\n    return actualTools.length === 1 && actualTools[0] === expectedTool ? 1 : 0;\n  }\n\n  return actualTools.includes(expectedTool) ? 1 : 0;\n}\n\nexport function createToolCallAccuracyScorerCode(options: ToolCallAccuracyOptions) {\n  const { expectedTool, strictMode = false, expectedToolOrder } = options;\n\n  if (!expectedTool && !expectedToolOrder) {\n    throw new Error('Either expectedTool or expectedToolOrder must be provided');\n  }\n\n  const getDescription = () => {\n    return expectedToolOrder\n      ? `Evaluates whether the LLM called tools in the correct order: [${expectedToolOrder.join(', ')}]`\n      : `Evaluates whether the LLM selected the correct tool (${expectedTool}) from the available tools`;\n  };\n\n  return createScorer<ScorerRunInputForAgent, ScorerRunOutputForAgent>({\n    name: 'Tool Call Accuracy',\n    description: getDescription(),\n  })\n    .preprocess(async ({ run }) => {\n      const isInputInvalid = !run.input || !run.input.inputMessages || run.input.inputMessages.length === 0;\n      const isOutputInvalid = !run.output || run.output.length === 0;\n\n      if (isInputInvalid || isOutputInvalid) {\n        throw new Error('Input and output messages cannot be null or empty');\n      }\n\n      const { tools: actualTools, toolCallInfos } = extractToolCalls(run.output);\n\n      const correctToolCalled = expectedTool\n        ? strictMode\n          ? actualTools.length === 1 && actualTools[0] === expectedTool\n          : actualTools.includes(expectedTool)\n        : false;\n\n      return {\n        expectedTool,\n        actualTools,\n        strictMode,\n        expectedToolOrder,\n        hasToolCalls: actualTools.length > 0,\n        correctToolCalled,\n        toolCallInfos,\n        correctOrderCalled: expectedToolOrder ? checkToolOrder(actualTools, expectedToolOrder, strictMode) : null,\n      };\n    })\n    .generateScore(({ results }) => {\n      const preprocessResult = results.preprocessStepResult;\n      if (!preprocessResult) {\n        return 0;\n      }\n\n      return calculateAccuracy({\n        expectedTool: preprocessResult.expectedTool,\n        actualTools: preprocessResult.actualTools,\n        strictMode: preprocessResult.strictMode,\n        expectedToolOrder: preprocessResult.expectedToolOrder,\n      });\n    });\n}\n"]}

package/dist/scorers/code/tool-call-accuracy/index.d.ts

@@ -0,0 +1,18 @@
+import type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '@mastra/core/scores';
+interface ToolCallAccuracyOptions {
+    expectedTool?: string;
+    strictMode?: boolean;
+    expectedToolOrder?: string[];
+}
+export declare function createToolCallAccuracyScorerCode(options: ToolCallAccuracyOptions): import("@mastra/core/scores").MastraScorer<string, ScorerRunInputForAgent, ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+    expectedTool: string | undefined;
+    actualTools: string[];
+    strictMode: boolean;
+    expectedToolOrder: string[] | undefined;
+    hasToolCalls: boolean;
+    correctToolCalled: boolean;
+    toolCallInfos: import("../../utils").ToolCallInfo[];
+    correctOrderCalled: boolean | null;
+}> & Record<"generateScoreStepResult", number>>;
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/scorers/code/tool-call-accuracy/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/code/tool-call-accuracy/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAI3F,UAAU,uBAAuB;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC9B;AAyDD,wBAAgB,gCAAgC,CAAC,OAAO,EAAE,uBAAuB;;;;;;;;;gDAyDhF"}