llm-testrunner-components 1.2.4 → 1.3.0

package/dist/collection/components/llm-test-runner/test-cases/llm-test-cases.js

@@ -1,7 +1,7 @@
 import { h } from "@stencil/core";
 import { LLMTestCaseRow } from "./llm-test-case-row";
 import { Button } from "../../../lib/ui/button/index";
-export const LLMTestCases = ({ testCases, dynamicResolutionSupported = false, onRun, onDelete, onAddTestCase, handleTestCaseChange, onExpectedOutcomeChange, onChatHistoryChange, }) => {
-    return (h("div", { class: "test-cases" }, h("div", { class: "test-cases__column-headers" }, h("div", { class: "test-cases__column-header" }, "Input"), h("div", { class: "test-cases__column-header" }, "Output"), h("div", { class: "test-cases__column-header" }, "Evaluation"), h("div", { class: "test-cases__column-header" }, "Actions")), testCases.map(testCase => (h(LLMTestCaseRow, { testCase: testCase, dynamicResolutionSupported: dynamicResolutionSupported, onRun: onRun, onDelete: onDelete, handleTestCaseChange: handleTestCaseChange, onExpectedOutcomeChange: onExpectedOutcomeChange, onChatHistoryChange: onChatHistoryChange }))), h("div", { class: "test-cases__add-section" }, h(Button, { variant: "outline", size: "md", onClick: onAddTestCase }, "+ Add Question"))));
+export const LLMTestCases = ({ testCases, dynamicResolutionSupported = false, extractorIds = [], onRun, onDelete, onAddTestCase, handleTestCaseChange, onExpectedOutcomeChange, onChatHistoryChange, }) => {
+    return (h("div", { class: "test-cases" }, h("div", { class: "test-cases__column-headers" }, h("div", { class: "test-cases__column-header" }, "Input"), h("div", { class: "test-cases__column-header" }, "Output"), h("div", { class: "test-cases__column-header" }, "Evaluation"), h("div", { class: "test-cases__column-header" }, "Actions")), testCases.map(testCase => (h(LLMTestCaseRow, { testCase: testCase, dynamicResolutionSupported: dynamicResolutionSupported, extractorIds: extractorIds, onRun: onRun, onDelete: onDelete, handleTestCaseChange: handleTestCaseChange, onExpectedOutcomeChange: onExpectedOutcomeChange, onChatHistoryChange: onChatHistoryChange }))), h("div", { class: "test-cases__add-section" }, h(Button, { variant: "outline", size: "md", onClick: onAddTestCase }, "+ Add Question"))));
 };
 //# sourceMappingURL=llm-test-cases.js.map

package/dist/collection/components/llm-test-runner/test-cases/llm-test-cases.js.map

@@ -1 +1 @@
-{"version":3,"file":"llm-test-cases.js","sourceRoot":"","sources":["../../../../src/components/llm-test-runner/test-cases/llm-test-cases.tsx"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAuB,MAAM,eAAe,CAAC;AAEvD,OAAO,EAAE,cAAc,EAA8B,MAAM,qBAAqB,CAAC;AACjF,OAAO,EAAE,MAAM,EAAE,MAAM,8BAA8B,CAAC;AAkBtD,MAAM,CAAC,MAAM,YAAY,GAA2C,CAAC,EACnE,SAAS,EACT,0BAA0B,GAAG,KAAK,EAClC,KAAK,EACL,QAAQ,EACR,aAAa,EACb,oBAAoB,EACpB,uBAAuB,EACvB,mBAAmB,GACpB,EAAE,EAAE;IACH,OAAO,CACL,WAAK,KAAK,EAAC,YAAY;QACrB,WAAK,KAAK,EAAC,4BAA4B;YACrC,WAAK,KAAK,EAAC,2BAA2B,YAAY;YAClD,WAAK,KAAK,EAAC,2BAA2B,aAAa;YACnD,WAAK,KAAK,EAAC,2BAA2B,iBAAiB;YACvD,WAAK,KAAK,EAAC,2BAA2B,cAAc,CAChD;QAEL,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CACzB,EAAC,cAAc,IACb,QAAQ,EAAE,QAAQ,EAClB,0BAA0B,EAAE,0BAA0B,EACtD,KAAK,EAAE,KAAK,EACZ,QAAQ,EAAE,QAAQ,EAClB,oBAAoB,EAAE,oBAAoB,EAC1C,uBAAuB,EAAE,uBAAuB,EAChD,mBAAmB,EAAE,mBAAmB,GACxC,CACH,CAAC;QAEF,WAAK,KAAK,EAAC,yBAAyB;YAClC,EAAC,MAAM,IAAC,OAAO,EAAC,SAAS,EAAC,IAAI,EAAC,IAAI,EAAC,OAAO,EAAE,aAAa,qBAEjD,CACL,CACF,CACP,CAAC;AACJ,CAAC,CAAC","sourcesContent":["import { h, FunctionalComponent } from '@stencil/core';\nimport { TestCase } from '../../../types/llm-test-runner';\nimport { LLMTestCaseRow, ChatHistoryRowChangeDetail } from './llm-test-case-row';\nimport { Button } from '../../../lib/ui/button/index';\nimport { ExpectedOutcomeChangeDetail } from './expected-outcome-renderer';\n\nexport interface LLMTestCasesProps {\n  testCases: TestCase[];\n  dynamicResolutionSupported?: boolean;\n  onRun: (testCase: TestCase) => void;\n  onDelete: (id: string) => void;\n  onAddTestCase: () => void;\n  handleTestCaseChange: (\n    e: CustomEvent<{ testCaseId: string; key: string; value: string }>,\n  ) => void;\n  onExpectedOutcomeChange: (\n    e: CustomEvent<ExpectedOutcomeChangeDetail>,\n  ) => void;\n  onChatHistoryChange: (e: CustomEvent<ChatHistoryRowChangeDetail>) => void;\n}\n\nexport const LLMTestCases: FunctionalComponent<LLMTestCasesProps> = ({\n  testCases,\n  dynamicResolutionSupported = false,\n  onRun,\n  onDelete,\n  onAddTestCase,\n  handleTestCaseChange,\n  onExpectedOutcomeChange,\n  onChatHistoryChange,\n}) => {\n  return (\n    <div class=\"test-cases\">\n      <div class=\"test-cases__column-headers\">\n        <div class=\"test-cases__column-header\">Input</div>\n        <div class=\"test-cases__column-header\">Output</div>\n        <div class=\"test-cases__column-header\">Evaluation</div>\n        <div class=\"test-cases__column-header\">Actions</div>\n      </div>\n\n      {testCases.map(testCase => (\n        <LLMTestCaseRow\n          testCase={testCase}\n          dynamicResolutionSupported={dynamicResolutionSupported}\n          onRun={onRun}\n          onDelete={onDelete}\n          handleTestCaseChange={handleTestCaseChange}\n          onExpectedOutcomeChange={onExpectedOutcomeChange}\n          onChatHistoryChange={onChatHistoryChange}\n        />\n      ))}\n\n      <div class=\"test-cases__add-section\">\n        <Button variant=\"outline\" size=\"md\" onClick={onAddTestCase}>\n          + Add Question\n        </Button>\n      </div>\n    </div>\n  );\n};\n"]}
+{"version":3,"file":"llm-test-cases.js","sourceRoot":"","sources":["../../../../src/components/llm-test-runner/test-cases/llm-test-cases.tsx"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAuB,MAAM,eAAe,CAAC;AAEvD,OAAO,EAAE,cAAc,EAA8B,MAAM,qBAAqB,CAAC;AACjF,OAAO,EAAE,MAAM,EAAE,MAAM,8BAA8B,CAAC;AAmBtD,MAAM,CAAC,MAAM,YAAY,GAA2C,CAAC,EACnE,SAAS,EACT,0BAA0B,GAAG,KAAK,EAClC,YAAY,GAAG,EAAE,EACjB,KAAK,EACL,QAAQ,EACR,aAAa,EACb,oBAAoB,EACpB,uBAAuB,EACvB,mBAAmB,GACpB,EAAE,EAAE;IACH,OAAO,CACL,WAAK,KAAK,EAAC,YAAY;QACrB,WAAK,KAAK,EAAC,4BAA4B;YACrC,WAAK,KAAK,EAAC,2BAA2B,YAAY;YAClD,WAAK,KAAK,EAAC,2BAA2B,aAAa;YACnD,WAAK,KAAK,EAAC,2BAA2B,iBAAiB;YACvD,WAAK,KAAK,EAAC,2BAA2B,cAAc,CAChD;QAEL,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CACzB,EAAC,cAAc,IACb,QAAQ,EAAE,QAAQ,EAClB,0BAA0B,EAAE,0BAA0B,EACtD,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,KAAK,EACZ,QAAQ,EAAE,QAAQ,EAClB,oBAAoB,EAAE,oBAAoB,EAC1C,uBAAuB,EAAE,uBAAuB,EAChD,mBAAmB,EAAE,mBAAmB,GACxC,CACH,CAAC;QAEF,WAAK,KAAK,EAAC,yBAAyB;YAClC,EAAC,MAAM,IAAC,OAAO,EAAC,SAAS,EAAC,IAAI,EAAC,IAAI,EAAC,OAAO,EAAE,aAAa,qBAEjD,CACL,CACF,CACP,CAAC;AACJ,CAAC,CAAC","sourcesContent":["import { h, FunctionalComponent } from '@stencil/core';\nimport { TestCase } from '../../../types/llm-test-runner';\nimport { LLMTestCaseRow, ChatHistoryRowChangeDetail } from './llm-test-case-row';\nimport { Button } from '../../../lib/ui/button/index';\nimport { ExpectedOutcomeChangeDetail } from './expected-outcome-renderer';\n\nexport interface LLMTestCasesProps {\n  testCases: TestCase[];\n  dynamicResolutionSupported?: boolean;\n  extractorIds?: string[];\n  onRun: (testCase: TestCase) => void;\n  onDelete: (id: string) => void;\n  onAddTestCase: () => void;\n  handleTestCaseChange: (\n    e: CustomEvent<{ testCaseId: string; key: string; value: string }>,\n  ) => void;\n  onExpectedOutcomeChange: (\n    e: CustomEvent<ExpectedOutcomeChangeDetail>,\n  ) => void;\n  onChatHistoryChange: (e: CustomEvent<ChatHistoryRowChangeDetail>) => void;\n}\n\nexport const LLMTestCases: FunctionalComponent<LLMTestCasesProps> = ({\n  testCases,\n  dynamicResolutionSupported = false,\n  extractorIds = [],\n  onRun,\n  onDelete,\n  onAddTestCase,\n  handleTestCaseChange,\n  onExpectedOutcomeChange,\n  onChatHistoryChange,\n}) => {\n  return (\n    <div class=\"test-cases\">\n      <div class=\"test-cases__column-headers\">\n        <div class=\"test-cases__column-header\">Input</div>\n        <div class=\"test-cases__column-header\">Output</div>\n        <div class=\"test-cases__column-header\">Evaluation</div>\n        <div class=\"test-cases__column-header\">Actions</div>\n      </div>\n\n      {testCases.map(testCase => (\n        <LLMTestCaseRow\n          testCase={testCase}\n          dynamicResolutionSupported={dynamicResolutionSupported}\n          extractorIds={extractorIds}\n          onRun={onRun}\n          onDelete={onDelete}\n          handleTestCaseChange={handleTestCaseChange}\n          onExpectedOutcomeChange={onExpectedOutcomeChange}\n          onChatHistoryChange={onChatHistoryChange}\n        />\n      ))}\n\n      <div class=\"test-cases__add-section\">\n        <Button variant=\"outline\" size=\"md\" onClick={onAddTestCase}>\n          + Add Question\n        </Button>\n      </div>\n    </div>\n  );\n};\n"]}

package/dist/collection/components/llm-test-runner/test-cases/output/response-output.js

@@ -1,5 +1,5 @@
 import { h } from "@stencil/core";
 export const ResponseOutput = ({ output, isRunning, }) => {
-    return (h("div", { class: "response-output" }, output ? (h("div", { class: "response-output__content" }, output)) : (h("div", { class: "response-output__placeholder" }, isRunning ? 'Running...' : ''))));
+    return (h("div", { class: "response-output" }, output?.text ? (h("div", { class: "response-output__content" }, output.text)) : (h("div", { class: "response-output__placeholder" }, isRunning ? 'Running...' : ''))));
 };
 //# sourceMappingURL=response-output.js.map

package/dist/collection/components/llm-test-runner/test-cases/output/response-output.js.map

@@ -1 +1 @@
-{"version":3,"file":"response-output.js","sourceRoot":"","sources":["../../../../../src/components/llm-test-runner/test-cases/output/response-output.tsx"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAuB,MAAM,eAAe,CAAC;AAOvD,MAAM,CAAC,MAAM,cAAc,GAA6C,CAAC,EACvE,MAAM,EACN,SAAS,GACV,EAAE,EAAE;IACH,OAAO,CACL,WAAK,KAAK,EAAC,iBAAiB,IACzB,MAAM,CAAC,CAAC,CAAC,CACR,WAAK,KAAK,EAAC,0BAA0B,IAAE,MAAM,CAAO,CACrD,CAAC,CAAC,CAAC,CACF,WAAK,KAAK,EAAC,8BAA8B,IAAE,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAO,CAChF,CACG,CACP,CAAC;AACJ,CAAC,CAAC","sourcesContent":["import { h, FunctionalComponent } from '@stencil/core';\n\nexport interface ResponseOutputProps {\n  output?: string;\n  isRunning: boolean;\n}\n\nexport const ResponseOutput: FunctionalComponent<ResponseOutputProps> = ({\n  output,\n  isRunning,\n}) => {\n  return (\n    <div class=\"response-output\">\n      {output ? (\n        <div class=\"response-output__content\">{output}</div>\n      ) : (\n        <div class=\"response-output__placeholder\">{isRunning ? 'Running...' : ''}</div>\n      )}\n    </div>\n  );\n};\n\n"]}
+{"version":3,"file":"response-output.js","sourceRoot":"","sources":["../../../../../src/components/llm-test-runner/test-cases/output/response-output.tsx"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAuB,MAAM,eAAe,CAAC;AAQvD,MAAM,CAAC,MAAM,cAAc,GAA6C,CAAC,EACvE,MAAM,EACN,SAAS,GACV,EAAE,EAAE;IACH,OAAO,CACL,WAAK,KAAK,EAAC,iBAAiB,IACzB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CACd,WAAK,KAAK,EAAC,0BAA0B,IAAE,MAAM,CAAC,IAAI,CAAO,CAC1D,CAAC,CAAC,CAAC,CACF,WAAK,KAAK,EAAC,8BAA8B,IAAE,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAO,CAChF,CACG,CACP,CAAC;AACJ,CAAC,CAAC","sourcesContent":["import { h, FunctionalComponent } from '@stencil/core';\nimport type { ModelResponsePayload } from '../../../../types/llm-test-runner';\n\nexport interface ResponseOutputProps {\n  output?: ModelResponsePayload;\n  isRunning: boolean;\n}\n\nexport const ResponseOutput: FunctionalComponent<ResponseOutputProps> = ({\n  output,\n  isRunning,\n}) => {\n  return (\n    <div class=\"response-output\">\n      {output?.text ? (\n        <div class=\"response-output__content\">{output.text}</div>\n      ) : (\n        <div class=\"response-output__placeholder\">{isRunning ? 'Running...' : ''}</div>\n      )}\n    </div>\n  );\n};\n\n"]}

package/dist/collection/lib/evaluation/actual-value-resolver.js

@@ -0,0 +1,52 @@
+function toTextSource() {
+    return { type: 'text' };
+}
+export async function resolveActualValue(field, output, extractors) {
+    const source = field.evaluationSource || toTextSource();
+    if (source.type === 'text') {
+        const text = output?.text?.trim();
+        if (!text) {
+            return {
+                success: false,
+                error: 'Model response text is empty.',
+            };
+        }
+        return { success: true, value: text };
+    }
+    const extractor = extractors?.[source.extractorId];
+    if (!extractor) {
+        return {
+            success: false,
+            error: `Extractor "${source.extractorId}" is not registered.`,
+        };
+    }
+    try {
+        const extractedRaw = await extractor(output || {});
+        if (typeof extractedRaw !== 'string') {
+            return {
+                success: false,
+                error: `Extractor "${source.extractorId}" must return a string.`,
+            };
+        }
+        const extracted = extractedRaw.trim();
+        if (!extracted) {
+            return {
+                success: false,
+                error: `Extractor "${source.extractorId}" returned an empty value.`,
+            };
+        }
+        return {
+            success: true,
+            value: extracted,
+        };
+    }
+    catch (error) {
+        return {
+            success: false,
+            error: error instanceof Error
+                ? error.message
+                : `Extractor "${source.extractorId}" failed.`,
+        };
+    }
+}
+//# sourceMappingURL=actual-value-resolver.js.map

package/dist/collection/lib/evaluation/actual-value-resolver.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"actual-value-resolver.js","sourceRoot":"","sources":["../../../src/lib/evaluation/actual-value-resolver.ts"],"names":[],"mappings":"AAUA,SAAS,YAAY;IACnB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAW,CAAC;AACnC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,KAA2B,EAC3B,MAA6B,EAC7B,UAAuC;IAEvC,MAAM,MAAM,GAAG,KAAK,CAAC,gBAAgB,IAAI,YAAY,EAAE,CAAC;IAExD,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,+BAA+B;aACvC,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IACxC,CAAC;IAED,MAAM,SAAS,GAAG,UAAU,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IACnD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,cAAc,MAAM,CAAC,WAAW,sBAAsB;SAC9D,CAAC;IACJ,CAAC;IAED,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QACnD,IAAI,OAAO,YAAY,KAAK,QAAQ,EAAE,CAAC;YACrC,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,cAAc,MAAM,CAAC,WAAW,yBAAyB;aACjE,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,YAAY,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,cAAc,MAAM,CAAC,WAAW,4BAA4B;aACpE,CAAC;QACJ,CAAC;QAED,OAAO;YACL,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,SAAS;SACjB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EACH,KAAK,YAAY,KAAK;gBACpB,CAAC,CAAC,KAAK,CAAC,OAAO;gBACf,CAAC,CAAC,cAAc,MAAM,CAAC,WAAW,WAAW;SAClD,CAAC;IACJ,CAAC;AACH,CAAC","sourcesContent":["import type {\n  EvaluationSourceExtractors,\n  ExpectedOutcomeField,\n  ModelResponsePayload,\n} from '../../types/llm-test-runner';\n\nexport type ResolvedActualValue =\n  | { success: true; value: string }\n  | { success: false; error: string };\n\nfunction toTextSource() {\n  return { type: 'text' } as const;\n}\n\nexport async function resolveActualValue(\n  field: ExpectedOutcomeField,\n  output?: ModelResponsePayload,\n  extractors?: EvaluationSourceExtractors,\n): Promise<ResolvedActualValue> {\n  const source = field.evaluationSource || toTextSource();\n\n  if (source.type === 'text') {\n    const text = output?.text?.trim();\n    if (!text) {\n      return {\n        success: false,\n        error: 'Model response text is empty.',\n      };\n    }\n    return { success: true, value: text };\n  }\n\n  const extractor = extractors?.[source.extractorId];\n  if (!extractor) {\n    return {\n      success: false,\n      error: `Extractor \"${source.extractorId}\" is not registered.`,\n    };\n  }\n\n  try {\n    const extractedRaw = await extractor(output || {});\n    if (typeof extractedRaw !== 'string') {\n      return {\n        success: false,\n        error: `Extractor \"${source.extractorId}\" must return a string.`,\n      };\n    }\n\n    const extracted = extractedRaw.trim();\n    if (!extracted) {\n      return {\n        success: false,\n        error: `Extractor \"${source.extractorId}\" returned an empty value.`,\n      };\n    }\n\n    return {\n      success: true,\n      value: extracted,\n    };\n  } catch (error) {\n    return {\n      success: false,\n      error:\n        error instanceof Error\n          ? error.message\n          : `Extractor \"${source.extractorId}\" failed.`,\n    };\n  }\n}\n"]}

package/dist/collection/lib/evaluation/evaluation-engine.js

@@ -10,7 +10,7 @@ export class LLMEvaluationEngine {
             const fieldRequest = {
                 testCaseId: request.testCaseId,
                 question: request.question,
-                actualResponse: request.actualResponse,
+                actualResponse: field.actualResponse,
                 expectedOutcome: field.expectedValue,
                 evaluationParameters: field.evaluationParameters,
             };

package/dist/collection/lib/evaluation/evaluation-engine.js.map

@@ -1 +1 @@
-{"version":3,"file":"evaluation-engine.js","sourceRoot":"","sources":["../../../src/lib/evaluation/evaluation-engine.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AACxE,OAAO,EAAE,yBAAyB,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AACxE,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AAEzE,MAAM,OAAO,mBAAmB;IAC9B,KAAK,CAAC,gBAAgB,CACpB,OAA4B,EAC5B,QAA4B;QAE5B,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,UAAU,CAC7C,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAC,KAAK,EAAC,EAAE;YAC/B,MAAM,YAAY,GAAsB;gBACtC,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,eAAe,EAAE,KAAK,CAAC,aAAa;gBACpC,oBAAoB,EAAE,KAAK,CAAC,oBAAoB;aACjD,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC;YAEtD,MAAM,WAAW,GAA0B;gBACzC,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,aAAa,EAAE,KAAK,CAAC,aAAa;gBAClC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,oBAAoB,EAAE,MAAM,CAAC,oBAAqB;gBAClD,wBAAwB,EAAE,MAAM,CAAC,wBAAwB;aAC1D,CAAC;YACF,OAAO,WAAW,CAAC;QACrB,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,YAAY,GAA4B,cAAc,CAAC,GAAG,CAC9D,CAAC,aAAa,EAAE,KAAK,EAAE,EAAE;YACvB,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACpC,IAAI,aAAa,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBACzC,OAAO,aAAa,CAAC,KAAK,CAAC;YAC7B,CAAC;YAED,OAAO;gBACL,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,aAAa,EAAE,KAAK,CAAC,aAAa;gBAClC,MAAM,EAAE,KAAK;gBACb,cAAc,EAAE,EAAE;gBAClB,oBAAoB,EAAE,KAAK,CAAC,oBAAoB;gBAChD,wBAAwB,EAAE;oBACxB,KAAK,EAAE,CAAC;oBACR,YAAY,EAAE,KAAK,CAAC,oBAAoB,CAAC,QAAQ;iBAClD;gBACD,KAAK,EAAE,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,MAAM,CAAC;aACtD,CAAC;QACJ,CAAC,CACF,CAAC;QAEF,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAC3E,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEzE,QAAQ,CAAC;YACP,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,MAAM;YACN,cAAc;YACd,YAAY;YACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAA0B;QACpD,MAAM,QAAQ,GAAuB,OAAO,CAAC,oBAAoB,CAAC,QAAQ,CAAC;QAC3E,QAAQ,QAAQ,EAAE,CAAC;YACjB,KAAK,kBAAkB,CAAC,IAAI;gBAC1B,OAAO,qBAAqB,CAAC,OAAO,CAAC,CAAC;YACxC,KAAK,kBAAkB,CAAC,KAAK;gBAC3B,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;YACpC,KAAK,kBAAkB,CAAC,OAAO;gBAC7B,OAAO,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAC1C,KAAK,kBAAkB,CAAC,OAAO;gBAC7B,OAAO,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAC1C,KAAK,kBAAkB,CAAC,QAAQ;gBAC9B,OAAO,yBAAyB,CAAC,OAAO,CAAC,CAAC;YAC5C;gBACE,OAAO,CAAC,IAAI,CACV,8BAA8B,OAAO,CAAC,oBAAoB,CAAC,QAAQ,kCAAkC,CACtG,CAAC;gBACF,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAEO,mBAAmB,CAAC,KAAc;QACxC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,0BAA0B,CAAC;IAC7E,CAAC;CACF","sourcesContent":["import {\n  EvaluationRequest,\n  EvaluationResult,\n  EvaluationCallback,\n  FieldEvaluationResult,\n  EvaluationRequestV2,\n} from './types';\nimport { performEvaluation } from './evaluators/exact/exact';\nimport { EvaluationApproach } from './constants';\nimport { performRouge1Evaluation } from './evaluators/rouge1-evaluator';\nimport { performSemanticEvaluation } from './evaluators/semantic/index';\nimport { performRougeLEvaluation } from './evaluators/rougeL-evaluator';\nimport { performBleuEvaluation } from './evaluators/bleu/bleu-evaluator';\n\nexport class LLMEvaluationEngine {\n  async evaluateResponse(\n    request: EvaluationRequestV2,\n    callback: EvaluationCallback,\n  ): Promise<void> {\n    const settledResults = await Promise.allSettled(\n      request.fields.map(async field => {\n        const fieldRequest: EvaluationRequest = {\n          testCaseId: request.testCaseId,\n          question: request.question,\n          actualResponse: request.actualResponse,\n          expectedOutcome: field.expectedValue,\n          evaluationParameters: field.evaluationParameters,\n        };\n        const result = await this.evaluateField(fieldRequest);\n\n        const fieldResult: FieldEvaluationResult = {\n          index: field.index,\n          label: field.label,\n          type: field.type,\n          expectedValue: field.expectedValue,\n          passed: result.passed,\n          keywordMatches: result.keywordMatches,\n          evaluationParameters: result.evaluationParameters!,\n          evaluationApproachResult: result.evaluationApproachResult,\n        };\n        return fieldResult;\n      }),\n    );\n\n    const fieldResults: FieldEvaluationResult[] = settledResults.map(\n      (settledResult, index) => {\n        const field = request.fields[index];\n        if (settledResult.status === 'fulfilled') {\n          return settledResult.value;\n        }\n\n        return {\n          index: field.index,\n          label: field.label,\n          type: field.type,\n          expectedValue: field.expectedValue,\n          passed: false,\n          keywordMatches: [],\n          evaluationParameters: field.evaluationParameters,\n          evaluationApproachResult: {\n            score: 0,\n            approachUsed: field.evaluationParameters.approach,\n          },\n          error: this.getSafeErrorMessage(settledResult.reason),\n        };\n      },\n    );\n\n    const keywordMatches = fieldResults.flatMap(field => field.keywordMatches);\n    const passed = fieldResults.every(field => field.passed && !field.error);\n\n    callback({\n      testCaseId: request.testCaseId,\n      passed,\n      keywordMatches,\n      fieldResults,\n      timestamp: new Date().toISOString(),\n    });\n  }\n\n  private async evaluateField(request: EvaluationRequest): Promise<EvaluationResult> {\n    const approach: EvaluationApproach = request.evaluationParameters.approach;\n    switch (approach) {\n      case EvaluationApproach.BLEU:\n        return performBleuEvaluation(request);\n      case EvaluationApproach.EXACT:\n        return performEvaluation(request);\n      case EvaluationApproach.ROUGE_1:\n        return performRouge1Evaluation(request);\n      case EvaluationApproach.ROUGE_L:\n        return performRougeLEvaluation(request);\n      case EvaluationApproach.SEMANTIC:\n        return performSemanticEvaluation(request);\n      default:\n        console.warn(\n          `Unknown matching approach: ${request.evaluationParameters.approach}, falling back to exact matching`,\n        );\n        return performEvaluation(request);\n    }\n  }\n\n  private getSafeErrorMessage(error: unknown): string {\n    return error instanceof Error ? error.message : 'Field evaluation failed.';\n  }\n}\n"]}
+{"version":3,"file":"evaluation-engine.js","sourceRoot":"","sources":["../../../src/lib/evaluation/evaluation-engine.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AACxE,OAAO,EAAE,yBAAyB,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AACxE,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AAEzE,MAAM,OAAO,mBAAmB;IAC9B,KAAK,CAAC,gBAAgB,CACpB,OAA4B,EAC5B,QAA4B;QAE5B,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,UAAU,CAC7C,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAC,KAAK,EAAC,EAAE;YAC/B,MAAM,YAAY,GAAsB;gBACtC,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,cAAc,EAAE,KAAK,CAAC,cAAc;gBACpC,eAAe,EAAE,KAAK,CAAC,aAAa;gBACpC,oBAAoB,EAAE,KAAK,CAAC,oBAAoB;aACjD,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC;YAEtD,MAAM,WAAW,GAA0B;gBACzC,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,aAAa,EAAE,KAAK,CAAC,aAAa;gBAClC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,oBAAoB,EAAE,MAAM,CAAC,oBAAqB;gBAClD,wBAAwB,EAAE,MAAM,CAAC,wBAAwB;aAC1D,CAAC;YACF,OAAO,WAAW,CAAC;QACrB,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,YAAY,GAA4B,cAAc,CAAC,GAAG,CAC9D,CAAC,aAAa,EAAE,KAAK,EAAE,EAAE;YACvB,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACpC,IAAI,aAAa,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBACzC,OAAO,aAAa,CAAC,KAAK,CAAC;YAC7B,CAAC;YAED,OAAO;gBACL,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,aAAa,EAAE,KAAK,CAAC,aAAa;gBAClC,MAAM,EAAE,KAAK;gBACb,cAAc,EAAE,EAAE;gBAClB,oBAAoB,EAAE,KAAK,CAAC,oBAAoB;gBAChD,wBAAwB,EAAE;oBACxB,KAAK,EAAE,CAAC;oBACR,YAAY,EAAE,KAAK,CAAC,oBAAoB,CAAC,QAAQ;iBAClD;gBACD,KAAK,EAAE,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,MAAM,CAAC;aACtD,CAAC;QACJ,CAAC,CACF,CAAC;QAEF,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAC3E,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEzE,QAAQ,CAAC;YACP,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,MAAM;YACN,cAAc;YACd,YAAY;YACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAA0B;QACpD,MAAM,QAAQ,GAAuB,OAAO,CAAC,oBAAoB,CAAC,QAAQ,CAAC;QAC3E,QAAQ,QAAQ,EAAE,CAAC;YACjB,KAAK,kBAAkB,CAAC,IAAI;gBAC1B,OAAO,qBAAqB,CAAC,OAAO,CAAC,CAAC;YACxC,KAAK,kBAAkB,CAAC,KAAK;gBAC3B,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;YACpC,KAAK,kBAAkB,CAAC,OAAO;gBAC7B,OAAO,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAC1C,KAAK,kBAAkB,CAAC,OAAO;gBAC7B,OAAO,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAC1C,KAAK,kBAAkB,CAAC,QAAQ;gBAC9B,OAAO,yBAAyB,CAAC,OAAO,CAAC,CAAC;YAC5C;gBACE,OAAO,CAAC,IAAI,CACV,8BAA8B,OAAO,CAAC,oBAAoB,CAAC,QAAQ,kCAAkC,CACtG,CAAC;gBACF,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAEO,mBAAmB,CAAC,KAAc;QACxC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,0BAA0B,CAAC;IAC7E,CAAC;CACF","sourcesContent":["import {\n  EvaluationRequest,\n  EvaluationResult,\n  EvaluationCallback,\n  FieldEvaluationResult,\n  EvaluationRequestV2,\n} from './types';\nimport { performEvaluation } from './evaluators/exact/exact';\nimport { EvaluationApproach } from './constants';\nimport { performRouge1Evaluation } from './evaluators/rouge1-evaluator';\nimport { performSemanticEvaluation } from './evaluators/semantic/index';\nimport { performRougeLEvaluation } from './evaluators/rougeL-evaluator';\nimport { performBleuEvaluation } from './evaluators/bleu/bleu-evaluator';\n\nexport class LLMEvaluationEngine {\n  async evaluateResponse(\n    request: EvaluationRequestV2,\n    callback: EvaluationCallback,\n  ): Promise<void> {\n    const settledResults = await Promise.allSettled(\n      request.fields.map(async field => {\n        const fieldRequest: EvaluationRequest = {\n          testCaseId: request.testCaseId,\n          question: request.question,\n          actualResponse: field.actualResponse,\n          expectedOutcome: field.expectedValue,\n          evaluationParameters: field.evaluationParameters,\n        };\n        const result = await this.evaluateField(fieldRequest);\n\n        const fieldResult: FieldEvaluationResult = {\n          index: field.index,\n          label: field.label,\n          type: field.type,\n          expectedValue: field.expectedValue,\n          passed: result.passed,\n          keywordMatches: result.keywordMatches,\n          evaluationParameters: result.evaluationParameters!,\n          evaluationApproachResult: result.evaluationApproachResult,\n        };\n        return fieldResult;\n      }),\n    );\n\n    const fieldResults: FieldEvaluationResult[] = settledResults.map(\n      (settledResult, index) => {\n        const field = request.fields[index];\n        if (settledResult.status === 'fulfilled') {\n          return settledResult.value;\n        }\n\n        return {\n          index: field.index,\n          label: field.label,\n          type: field.type,\n          expectedValue: field.expectedValue,\n          passed: false,\n          keywordMatches: [],\n          evaluationParameters: field.evaluationParameters,\n          evaluationApproachResult: {\n            score: 0,\n            approachUsed: field.evaluationParameters.approach,\n          },\n          error: this.getSafeErrorMessage(settledResult.reason),\n        };\n      },\n    );\n\n    const keywordMatches = fieldResults.flatMap(field => field.keywordMatches);\n    const passed = fieldResults.every(field => field.passed && !field.error);\n\n    callback({\n      testCaseId: request.testCaseId,\n      passed,\n      keywordMatches,\n      fieldResults,\n      timestamp: new Date().toISOString(),\n    });\n  }\n\n  private async evaluateField(request: EvaluationRequest): Promise<EvaluationResult> {\n    const approach: EvaluationApproach = request.evaluationParameters.approach;\n    switch (approach) {\n      case EvaluationApproach.BLEU:\n        return performBleuEvaluation(request);\n      case EvaluationApproach.EXACT:\n        return performEvaluation(request);\n      case EvaluationApproach.ROUGE_1:\n        return performRouge1Evaluation(request);\n      case EvaluationApproach.ROUGE_L:\n        return performRougeLEvaluation(request);\n      case EvaluationApproach.SEMANTIC:\n        return performSemanticEvaluation(request);\n      default:\n        console.warn(\n          `Unknown matching approach: ${request.evaluationParameters.approach}, falling back to exact matching`,\n        );\n        return performEvaluation(request);\n    }\n  }\n\n  private getSafeErrorMessage(error: unknown): string {\n    return error instanceof Error ? error.message : 'Field evaluation failed.';\n  }\n}\n"]}

package/dist/collection/lib/evaluation/evaluation-service.js

@@ -1,5 +1,6 @@
 import { LLMEvaluationEngine } from "./evaluation-engine";
 import { normalizeEvaluationParametersForField } from "./field-evaluation-approach";
+import { resolveActualValue } from "./actual-value-resolver";
 /**
  * Service for evaluating test case responses
  */
@@ -13,34 +14,71 @@ export class EvaluationService {
      * @param testCase - The test case to evaluate
      * @param onResult - Callback to handle the evaluation result
      */
-    async evaluateTestCase(testCase, onResult) {
-        if (!testCase.output) {
-            console.warn('⚠️ No output to evaluate for test case:', testCase.id);
-            return;
-        }
-        const fields = (testCase.expectedOutcome || []).flatMap((field, index) => {
+    async evaluateTestCase(testCase, onResult, extractors) {
+        const fields = [];
+        const failedFields = [];
+        for (const [index, field] of (testCase.expectedOutcome || []).entries()) {
             if (field.type === 'textarea' && field.outcomeMode === 'dynamic') {
-                return [];
+                continue;
             }
-            return [
-                {
+            const evaluationParameters = normalizeEvaluationParametersForField(field.type, field.evaluationParameters);
+            const expectedValue = getFieldExpectedValue(field);
+            const resolvedActualValue = await resolveActualValue(field, testCase.output, extractors);
+            if (resolvedActualValue.success) {
+                fields.push({
                     index,
                     label: field.label,
                     type: field.type,
-                    expectedValue: getFieldExpectedValue(field),
-                    evaluationParameters: normalizeEvaluationParametersForField(field.type, field.evaluationParameters),
-                },
-            ];
-        });
+                    expectedValue,
+                    actualResponse: resolvedActualValue.value,
+                    evaluationParameters,
+                });
+            }
+            else {
+                failedFields.push({
+                    index,
+                    label: field.label,
+                    type: field.type,
+                    expectedValue,
+                    passed: false,
+                    keywordMatches: [],
+                    evaluationParameters,
+                    evaluationApproachResult: {
+                        score: 0,
+                        approachUsed: evaluationParameters.approach,
+                    },
+                    error: 'error' in resolvedActualValue
+                        ? resolvedActualValue.error
+                        : 'Failed to resolve actual value.',
+                });
+            }
+        }
+        if (fields.length === 0) {
+            if (failedFields.length === 0) {
+                console.warn('⚠️ No evaluable fields for test case:', testCase.id);
+                return;
+            }
+            onResult({
+                testCaseId: testCase.id,
+                passed: false,
+                keywordMatches: [],
+                fieldResults: failedFields,
+                timestamp: new Date().toISOString(),
+            });
+            return;
+        }
         const evaluationRequest = {
             testCaseId: testCase.id,
             question: testCase.question,
-            actualResponse: testCase.output,
             fields,
         };
         await this.engine.evaluateResponse(evaluationRequest, (result) => {
-            console.log('📊 Evaluation result received:', result);
-            onResult(result);
+            const combinedResults = [...(result.fieldResults || []), ...failedFields].sort((a, b) => a.index - b.index);
+            onResult({
+                ...result,
+                passed: combinedResults.every(field => field.passed && !field.error),
+                fieldResults: combinedResults,
+            });
         });
     }
 }

package/dist/collection/lib/evaluation/evaluation-service.js.map

@@ -1 +1 @@
-{"version":3,"file":"evaluation-service.js","sourceRoot":"","sources":["../../../src/lib/evaluation/evaluation-service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAO1D,OAAO,EAAE,qCAAqC,EAAE,MAAM,6BAA6B,CAAC;AAEpF;;GAEG;AACH,MAAM,OAAO,iBAAiB;IACpB,MAAM,CAAsB;IAEpC;QACE,IAAI,CAAC,MAAM,GAAG,IAAI,mBAAmB,EAAE,CAAC;IAC1C,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB,CACpB,QAAkB,EAClB,QAA4C;QAE5C,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,yCAAyC,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;YACrE,OAAO;QACT,CAAC;QAED,MAAM,MAAM,GAA2B,CAAC,QAAQ,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,OAAO,CAC7E,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACf,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;gBACjE,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,OAAO;gBACL;oBACE,KAAK;oBACL,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,aAAa,EAAE,qBAAqB,CAAC,KAAK,CAAC;oBAC3C,oBAAoB,EAAE,qCAAqC,CACzD,KAAK,CAAC,IAAI,EACV,KAAK,CAAC,oBAAoB,CAC3B;iBACF;aACF,CAAC;QACJ,CAAC,CACF,CAAC;QAEF,MAAM,iBAAiB,GAAwB;YAC7C,UAAU,EAAE,QAAQ,CAAC,EAAE;YACvB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,cAAc,EAAE,QAAQ,CAAC,MAAM;YAC/B,MAAM;SACP,CAAC;QAEF,MAAM,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAChC,iBAAiB,EACjB,CAAC,MAAwB,EAAE,EAAE;YAC3B,OAAO,CAAC,GAAG,CAAC,gCAAgC,EAAE,MAAM,CAAC,CAAC;YACtD,QAAQ,CAAC,MAAM,CAAC,CAAC;QACnB,CAAC,CACF,CAAC;IACJ,CAAC;CACF;AAED,SAAS,qBAAqB,CAAC,KAA2B;IACxD,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;QACjC,OAAO,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,KAAK,CAAC,KAAK,CAAC;AACrB,CAAC","sourcesContent":["import { LLMEvaluationEngine } from './evaluation-engine';\nimport {\n  EvaluationResult,\n  FieldEvaluationInput,\n  EvaluationRequestV2,\n} from './types';\nimport { TestCase, ExpectedOutcomeField } from '../../types/llm-test-runner';\nimport { normalizeEvaluationParametersForField } from './field-evaluation-approach';\n\n/**\n * Service for evaluating test case responses\n */\nexport class EvaluationService {\n  private engine: LLMEvaluationEngine;\n\n  constructor() {\n    this.engine = new LLMEvaluationEngine();\n  }\n\n  /**\n   * Evaluates a test case response\n   * @param testCase - The test case to evaluate\n   * @param onResult - Callback to handle the evaluation result\n   */\n  async evaluateTestCase(\n    testCase: TestCase,\n    onResult: (result: EvaluationResult) => void,\n  ): Promise<void> {\n    if (!testCase.output) {\n      console.warn('⚠️ No output to evaluate for test case:', testCase.id);\n      return;\n    }\n\n    const fields: FieldEvaluationInput[] = (testCase.expectedOutcome || []).flatMap(\n      (field, index) => {\n        if (field.type === 'textarea' && field.outcomeMode === 'dynamic') {\n          return [];\n        }\n\n        return [\n          {\n            index,\n            label: field.label,\n            type: field.type,\n            expectedValue: getFieldExpectedValue(field),\n            evaluationParameters: normalizeEvaluationParametersForField(\n              field.type,\n              field.evaluationParameters,\n            ),\n          },\n        ];\n      },\n    );\n\n    const evaluationRequest: EvaluationRequestV2 = {\n      testCaseId: testCase.id,\n      question: testCase.question,\n      actualResponse: testCase.output,\n      fields,\n    };\n\n    await this.engine.evaluateResponse(\n      evaluationRequest,\n      (result: EvaluationResult) => {\n        console.log('📊 Evaluation result received:', result);\n        onResult(result);\n      },\n    );\n  }\n}\n\nfunction getFieldExpectedValue(field: ExpectedOutcomeField): string {\n  if (field.type === 'chips-input') {\n    return field.value.join(', ');\n  }\n  return field.value;\n}\n"]}
+{"version":3,"file":"evaluation-service.js","sourceRoot":"","sources":["../../../src/lib/evaluation/evaluation-service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAY1D,OAAO,EAAE,qCAAqC,EAAE,MAAM,6BAA6B,CAAC;AACpF,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAE7D;;GAEG;AACH,MAAM,OAAO,iBAAiB;IACpB,MAAM,CAAsB;IAEpC;QACE,IAAI,CAAC,MAAM,GAAG,IAAI,mBAAmB,EAAE,CAAC;IAC1C,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB,CACpB,QAAkB,EAClB,QAA4C,EAC5C,UAAuC;QAEvC,MAAM,MAAM,GAA2B,EAAE,CAAC;QAC1C,MAAM,YAAY,GAA4B,EAAE,CAAC;QAEjD,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;YACxE,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;gBACjE,SAAS;YACX,CAAC;YAED,MAAM,oBAAoB,GAAG,qCAAqC,CAChE,KAAK,CAAC,IAAI,EACV,KAAK,CAAC,oBAAoB,CAC3B,CAAC;YACF,MAAM,aAAa,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;YACnD,MAAM,mBAAmB,GAAG,MAAM,kBAAkB,CAClD,KAAK,EACL,QAAQ,CAAC,MAAM,EACf,UAAU,CACX,CAAC;YAEF,IAAI,mBAAmB,CAAC,OAAO,EAAE,CAAC;gBAChC,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK;oBACL,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,aAAa;oBACb,cAAc,EAAE,mBAAmB,CAAC,KAAK;oBACzC,oBAAoB;iBACrB,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,YAAY,CAAC,IAAI,CAAC;oBAChB,KAAK;oBACL,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,aAAa;oBACb,MAAM,EAAE,KAAK;oBACb,cAAc,EAAE,EAAE;oBAClB,oBAAoB;oBACpB,wBAAwB,EAAE;wBACxB,KAAK,EAAE,CAAC;wBACR,YAAY,EAAE,oBAAoB,CAAC,QAAQ;qBAC5C;oBACD,KAAK,EACH,OAAO,IAAI,mBAAmB;wBAC5B,CAAC,CAAC,mBAAmB,CAAC,KAAK;wBAC3B,CAAC,CAAC,iCAAiC;iBACxC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9B,OAAO,CAAC,IAAI,CAAC,uCAAuC,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACnE,OAAO;YACT,CAAC;YAED,QAAQ,CAAC;gBACP,UAAU,EAAE,QAAQ,CAAC,EAAE;gBACvB,MAAM,EAAE,KAAK;gBACb,cAAc,EAAE,EAAE;gBAClB,YAAY,EAAE,YAAY;gBAC1B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,MAAM,iBAAiB,GAAwB;YAC7C,UAAU,EAAE,QAAQ,CAAC,EAAE;YACvB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,MAAM;SACP,CAAC;QAEF,MAAM,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,CAAC,MAAwB,EAAE,EAAE;YACjF,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,CAAC,EAAE,GAAG,YAAY,CAAC,CAAC,IAAI,CAC5E,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAC5B,CAAC;YACF,QAAQ,CAAC;gBACP,GAAG,MAAM;gBACT,MAAM,EAAE,eAAe,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;gBACpE,YAAY,EAAE,eAAe;aAC9B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AAED,SAAS,qBAAqB,CAAC,KAA2B;IACxD,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;QACjC,OAAO,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,KAAK,CAAC,KAAK,CAAC;AACrB,CAAC","sourcesContent":["import { LLMEvaluationEngine } from './evaluation-engine';\nimport {\n  EvaluationResult,\n  FieldEvaluationInput,\n  EvaluationRequestV2,\n  FieldEvaluationResult,\n} from './types';\nimport {\n  TestCase,\n  ExpectedOutcomeField,\n  EvaluationSourceExtractors,\n} from '../../types/llm-test-runner';\nimport { normalizeEvaluationParametersForField } from './field-evaluation-approach';\nimport { resolveActualValue } from './actual-value-resolver';\n\n/**\n * Service for evaluating test case responses\n */\nexport class EvaluationService {\n  private engine: LLMEvaluationEngine;\n\n  constructor() {\n    this.engine = new LLMEvaluationEngine();\n  }\n\n  /**\n   * Evaluates a test case response\n   * @param testCase - The test case to evaluate\n   * @param onResult - Callback to handle the evaluation result\n   */\n  async evaluateTestCase(\n    testCase: TestCase,\n    onResult: (result: EvaluationResult) => void,\n    extractors?: EvaluationSourceExtractors,\n  ): Promise<void> {\n    const fields: FieldEvaluationInput[] = [];\n    const failedFields: FieldEvaluationResult[] = [];\n\n    for (const [index, field] of (testCase.expectedOutcome || []).entries()) {\n      if (field.type === 'textarea' && field.outcomeMode === 'dynamic') {\n        continue;\n      }\n\n      const evaluationParameters = normalizeEvaluationParametersForField(\n        field.type,\n        field.evaluationParameters,\n      );\n      const expectedValue = getFieldExpectedValue(field);\n      const resolvedActualValue = await resolveActualValue(\n        field,\n        testCase.output,\n        extractors,\n      );\n\n      if (resolvedActualValue.success) {\n        fields.push({\n          index,\n          label: field.label,\n          type: field.type,\n          expectedValue,\n          actualResponse: resolvedActualValue.value,\n          evaluationParameters,\n        });\n      } else {\n        failedFields.push({\n          index,\n          label: field.label,\n          type: field.type,\n          expectedValue,\n          passed: false,\n          keywordMatches: [],\n          evaluationParameters,\n          evaluationApproachResult: {\n            score: 0,\n            approachUsed: evaluationParameters.approach,\n          },\n          error:\n            'error' in resolvedActualValue\n              ? resolvedActualValue.error\n              : 'Failed to resolve actual value.',\n        });\n      }\n    }\n\n    if (fields.length === 0) {\n      if (failedFields.length === 0) {\n        console.warn('⚠️ No evaluable fields for test case:', testCase.id);\n        return;\n      }\n\n      onResult({\n        testCaseId: testCase.id,\n        passed: false,\n        keywordMatches: [],\n        fieldResults: failedFields,\n        timestamp: new Date().toISOString(),\n      });\n      return;\n    }\n\n    const evaluationRequest: EvaluationRequestV2 = {\n      testCaseId: testCase.id,\n      question: testCase.question,\n      fields,\n    };\n\n    await this.engine.evaluateResponse(evaluationRequest, (result: EvaluationResult) => {\n      const combinedResults = [...(result.fieldResults || []), ...failedFields].sort(\n        (a, b) => a.index - b.index,\n      );\n      onResult({\n        ...result,\n        passed: combinedResults.every(field => field.passed && !field.error),\n        fieldResults: combinedResults,\n      });\n    });\n  }\n}\n\nfunction getFieldExpectedValue(field: ExpectedOutcomeField): string {\n  if (field.type === 'chips-input') {\n    return field.value.join(', ');\n  }\n  return field.value;\n}\n"]}

package/dist/collection/lib/evaluation/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/lib/evaluation/types.ts"],"names":[],"mappings":"","sourcesContent":["import {\n  EvaluationParameters,\n  EvaluationApproachResult,\n} from '../../types/evaluation';\nimport type { ExpectedOutcomeFieldType } from '../../types/llm-test-runner';\n\nexport interface EvaluationRequest {\n  testCaseId: string;\n  question: string;\n  expectedOutcome: string;\n  actualResponse: string;\n  evaluationParameters: EvaluationParameters;\n}\n\nexport interface FieldEvaluationInput {\n  index: number;\n  label: string;\n  type: ExpectedOutcomeFieldType;\n  expectedValue: string;\n  evaluationParameters: EvaluationParameters;\n}\n\nexport interface EvaluationRequestV2 {\n  testCaseId: string;\n  question: string;\n  actualResponse: string;\n  fields: FieldEvaluationInput[];\n}\n\nexport interface EvaluationResult {\n  testCaseId: string;\n  passed: boolean;\n  keywordMatches: KeywordMatch[];\n  fieldResults?: FieldEvaluationResult[];\n  timestamp?: string;\n  evaluationParameters?: EvaluationParameters;\n  evaluationApproachResult?: EvaluationApproachResult;\n}\n\nexport interface FieldEvaluationResult {\n  index: number;\n  label: string;\n  type: ExpectedOutcomeFieldType;\n  expectedValue: string;\n  passed: boolean;\n  keywordMatches: KeywordMatch[];\n  evaluationParameters: EvaluationParameters;\n  evaluationApproachResult: EvaluationApproachResult;\n  error?: string;\n}\n\nexport interface KeywordMatch {\n  keyword: string;\n  found: boolean;\n  evaluationApproachResult: EvaluationApproachResult;\n}\n\nexport type EvaluationCallback = (result: EvaluationResult) => void;\n\nexport interface RougeKeywordDetails {\n  rouge1: number;\n  rougeL: number;\n  scoreUsed: string;\n  approach: string;\n}\n\nexport interface Rouge1OverallDetails {\n  keywordsPassed: number;\n  totalKeywords: number;\n  passRate: string;\n  thresholdUsed: number;\n  approach: string;\n}\n"]}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/lib/evaluation/types.ts"],"names":[],"mappings":"","sourcesContent":["import {\n  EvaluationParameters,\n  EvaluationApproachResult,\n} from '../../types/evaluation';\nimport type { ExpectedOutcomeFieldType } from '../../types/llm-test-runner';\n\nexport interface EvaluationRequest {\n  testCaseId: string;\n  question: string;\n  expectedOutcome: string;\n  actualResponse: string;\n  evaluationParameters: EvaluationParameters;\n}\n\nexport interface FieldEvaluationInput {\n  index: number;\n  label: string;\n  type: ExpectedOutcomeFieldType;\n  expectedValue: string;\n  actualResponse: string;\n  evaluationParameters: EvaluationParameters;\n}\n\nexport interface EvaluationRequestV2 {\n  testCaseId: string;\n  question: string;\n  fields: FieldEvaluationInput[];\n}\n\nexport interface EvaluationResult {\n  testCaseId: string;\n  passed: boolean;\n  keywordMatches: KeywordMatch[];\n  fieldResults?: FieldEvaluationResult[];\n  timestamp?: string;\n  evaluationParameters?: EvaluationParameters;\n  evaluationApproachResult?: EvaluationApproachResult;\n}\n\nexport interface FieldEvaluationResult {\n  index: number;\n  label: string;\n  type: ExpectedOutcomeFieldType;\n  expectedValue: string;\n  passed: boolean;\n  keywordMatches: KeywordMatch[];\n  evaluationParameters: EvaluationParameters;\n  evaluationApproachResult: EvaluationApproachResult;\n  error?: string;\n}\n\nexport interface KeywordMatch {\n  keyword: string;\n  found: boolean;\n  evaluationApproachResult: EvaluationApproachResult;\n}\n\nexport type EvaluationCallback = (result: EvaluationResult) => void;\n\nexport interface RougeKeywordDetails {\n  rouge1: number;\n  rougeL: number;\n  scoreUsed: string;\n  approach: string;\n}\n\nexport interface Rouge1OverallDetails {\n  keywordsPassed: number;\n  totalKeywords: number;\n  passRate: string;\n  thresholdUsed: number;\n  approach: string;\n}\n"]}

package/dist/collection/lib/import-export/test-suite-importer.js

@@ -1,14 +1,20 @@
 import { createTestCaseFromInput } from "../test-cases/test-case-factory";
 import { validateTestCaseInputArray } from "../../schemas/test-case";
+import { validateExpectedOutcomeArrayWithExtractors } from "../../schemas/expected-outcome";
 /**
  * Validates and imports test cases from JSON content
  * @param jsonContent - The JSON string to parse and validate
  * @returns Validation result with test cases or error message
  */
-export function importTestSuite(jsonContent) {
+export function importTestSuite(jsonContent, allowedExtractorIds = []) {
     try {
         const parsed = JSON.parse(jsonContent);
         validateTestCaseInputArray(parsed);
+        if (allowedExtractorIds.length > 0) {
+            parsed.forEach((testCase) => {
+                validateExpectedOutcomeArrayWithExtractors(testCase.expectedOutcome, allowedExtractorIds);
+            });
+        }
         const testCases = parsed.map((item, index) => {
             try {
                 return createTestCaseFromInput(item);

package/dist/collection/lib/import-export/test-suite-importer.js.map

@@ -1 +1 @@
-{"version":3,"file":"test-suite-importer.js","sourceRoot":"","sources":["../../../src/lib/import-export/test-suite-importer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAC1E,OAAO,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;AAQrE;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,WAAmB;IACjD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACvC,0BAA0B,CAAC,MAAM,CAAC,CAAC;QAEnC,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YAC3C,IAAI,CAAC;gBACH,OAAO,uBAAuB,CAAC,IAAI,CAAC,CAAC;YACvC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;gBACrE,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,KAAK,OAAO,EAAE,CAAC,CAAC;YACrE,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,SAAS;SACV,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EACH,GAAG,YAAY,KAAK;gBAClB,CAAC,CAAC,GAAG,CAAC,OAAO;gBACb,CAAC,CAAC,gEAAgE;SACvE,CAAC;IACJ,CAAC;AACH,CAAC","sourcesContent":["import type { TestCase } from '../../types/llm-test-runner';\nimport { createTestCaseFromInput } from '../test-cases/test-case-factory';\nimport { validateTestCaseInputArray } from '../../schemas/test-case';\n\nexport interface ImportValidationResult {\n  success: boolean;\n  testCases?: TestCase[];\n  error?: string;\n}\n\n/**\n * Validates and imports test cases from JSON content\n * @param jsonContent - The JSON string to parse and validate\n * @returns Validation result with test cases or error message\n */\nexport function importTestSuite(jsonContent: string): ImportValidationResult {\n  try {\n    const parsed = JSON.parse(jsonContent);\n    validateTestCaseInputArray(parsed);\n\n    const testCases = parsed.map((item, index) => {\n      try {\n        return createTestCaseFromInput(item);\n      } catch (err) {\n        const message = err instanceof Error ? err.message : 'Unknown error';\n        throw new Error(`Invalid test case at index ${index}: ${message}`);\n      }\n    });\n\n    return {\n      success: true,\n      testCases,\n    };\n  } catch (err) {\n    return {\n      success: false,\n      error:\n        err instanceof Error\n          ? err.message\n          : 'Error processing file. Please ensure it is a valid JSON array.',\n    };\n  }\n}\n\n"]}
+{"version":3,"file":"test-suite-importer.js","sourceRoot":"","sources":["../../../src/lib/import-export/test-suite-importer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAC1E,OAAO,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;AACrE,OAAO,EAAE,0CAA0C,EAAE,MAAM,gCAAgC,CAAC;AAQ5F;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAC7B,WAAmB,EACnB,sBAAgC,EAAE;IAElC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACvC,0BAA0B,CAAC,MAAM,CAAC,CAAC;QACnC,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;gBAC1B,0CAA0C,CACxC,QAAQ,CAAC,eAAe,EACxB,mBAAmB,CACpB,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YAC3C,IAAI,CAAC;gBACH,OAAO,uBAAuB,CAAC,IAAI,CAAC,CAAC;YACvC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;gBACrE,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,KAAK,OAAO,EAAE,CAAC,CAAC;YACrE,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,SAAS;SACV,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EACH,GAAG,YAAY,KAAK;gBAClB,CAAC,CAAC,GAAG,CAAC,OAAO;gBACb,CAAC,CAAC,gEAAgE;SACvE,CAAC;IACJ,CAAC;AACH,CAAC","sourcesContent":["import type { TestCase } from '../../types/llm-test-runner';\nimport { createTestCaseFromInput } from '../test-cases/test-case-factory';\nimport { validateTestCaseInputArray } from '../../schemas/test-case';\nimport { validateExpectedOutcomeArrayWithExtractors } from '../../schemas/expected-outcome';\n\nexport interface ImportValidationResult {\n  success: boolean;\n  testCases?: TestCase[];\n  error?: string;\n}\n\n/**\n * Validates and imports test cases from JSON content\n * @param jsonContent - The JSON string to parse and validate\n * @returns Validation result with test cases or error message\n */\nexport function importTestSuite(\n  jsonContent: string,\n  allowedExtractorIds: string[] = [],\n): ImportValidationResult {\n  try {\n    const parsed = JSON.parse(jsonContent);\n    validateTestCaseInputArray(parsed);\n    if (allowedExtractorIds.length > 0) {\n      parsed.forEach((testCase) => {\n        validateExpectedOutcomeArrayWithExtractors(\n          testCase.expectedOutcome,\n          allowedExtractorIds,\n        );\n      });\n    }\n\n    const testCases = parsed.map((item, index) => {\n      try {\n        return createTestCaseFromInput(item);\n      } catch (err) {\n        const message = err instanceof Error ? err.message : 'Unknown error';\n        throw new Error(`Invalid test case at index ${index}: ${message}`);\n      }\n    });\n\n    return {\n      success: true,\n      testCases,\n    };\n  } catch (err) {\n    return {\n      success: false,\n      error:\n        err instanceof Error\n          ? err.message\n          : 'Error processing file. Please ensure it is a valid JSON array.',\n    };\n  }\n}\n\n"]}

package/dist/collection/lib/test-cases/test-case-factory.js

@@ -11,6 +11,7 @@ export const DEFAULT_EXPECTED_OUTCOME_SCHEMA = [
 function normalizeExpectedOutcomeField(field) {
     return {
         ...field,
+        evaluationSource: field.evaluationSource || { type: 'text' },
         evaluationParameters: normalizeEvaluationParametersForField(field.type, field.evaluationParameters),
     };
 }
@@ -34,6 +35,7 @@ function createExpectedOutcomeFieldFromSchema(schemaField) {
                 type: 'text',
                 label: schemaField.label,
                 placeholder: schemaField.placeholder,
+                evaluationSource: schemaField.evaluationSource || { type: 'text' },
                 value: '',
                 evaluationParameters: normalizeEvaluationParametersForField(schemaField.type, schemaField.evaluationParameters),
             };
@@ -42,6 +44,7 @@ function createExpectedOutcomeFieldFromSchema(schemaField) {
                 type: 'textarea',
                 label: schemaField.label,
                 placeholder: schemaField.placeholder,
+                evaluationSource: schemaField.evaluationSource || { type: 'text' },
                 rows: schemaField.rows,
                 value: '',
                 evaluationParameters: normalizeEvaluationParametersForField(schemaField.type, schemaField.evaluationParameters),
@@ -51,6 +54,7 @@ function createExpectedOutcomeFieldFromSchema(schemaField) {
                 type: 'chips-input',
                 label: schemaField.label,
                 placeholder: schemaField.placeholder,
+                evaluationSource: schemaField.evaluationSource || { type: 'text' },
                 value: [],
                 evaluationParameters: normalizeEvaluationParametersForField(schemaField.type, schemaField.evaluationParameters),
             };
@@ -59,6 +63,7 @@ function createExpectedOutcomeFieldFromSchema(schemaField) {
                 type: 'select',
                 label: schemaField.label,
                 placeholder: schemaField.placeholder,
+                evaluationSource: schemaField.evaluationSource || { type: 'text' },
                 value: schemaField.options[0],
                 options: schemaField.options,
                 evaluationParameters: normalizeEvaluationParametersForField(schemaField.type, schemaField.evaluationParameters),

package/dist/collection/lib/test-cases/test-case-factory.js.map

@@ -1 +1 @@
-{"version":3,"file":"test-case-factory.js","sourceRoot":"","sources":["../../../src/lib/test-cases/test-case-factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AASpC,OAAO,EAAE,qCAAqC,EAAE,MAAM,yCAAyC,CAAC;AAEhG,MAAM,CAAC,MAAM,+BAA+B,GAA0B;IACpE;QACE,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,kBAAkB;QACzB,WAAW,EAAE,2BAA2B;QACxC,IAAI,EAAE,CAAC;KACR;CACF,CAAC;AAEF,SAAS,6BAA6B,CACpC,KAA2B;IAE3B,OAAO;QACL,GAAG,KAAK;QACR,oBAAoB,EAAE,qCAAqC,CACzD,KAAK,CAAC,IAAI,EACV,KAAK,CAAC,oBAAoB,CAC3B;KACF,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAC5B,wBAA+C,+BAA+B;IAE9E,OAAO;QACL,EAAE,EAAE,MAAM,EAAE;QACZ,QAAQ,EAAE,EAAE;QACZ,eAAe,EAAE,+BAA+B,CAAC,qBAAqB,CAAC;QACvE,WAAW,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE;QAC1C,SAAS,EAAE,KAAK;KACjB,CAAC;AACJ,CAAC;AAED,SAAS,oCAAoC,CAC3C,WAAuC;IAEvC,QAAQ,WAAW,CAAC,IAAI,EAAE,CAAC;QACzB,KAAK,MAAM;YACT,OAAO;gBACL,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,WAAW,EAAE,WAAW,CAAC,WAAW;gBACpC,KAAK,EAAE,EAAE;gBACT,oBAAoB,EAAE,qCAAqC,CACzD,WAAW,CAAC,IAAI,EAChB,WAAW,CAAC,oBAAoB,CACjC;aACF,CAAC;QAEJ,KAAK,UAAU;YACb,OAAO;gBACL,IAAI,EAAE,UAAU;gBAChB,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,WAAW,EAAE,WAAW,CAAC,WAAW;gBACpC,IAAI,EAAE,WAAW,CAAC,IAAI;gBACtB,KAAK,EAAE,EAAE;gBACT,oBAAoB,EAAE,qCAAqC,CACzD,WAAW,CAAC,IAAI,EAChB,WAAW,CAAC,oBAAoB,CACjC;aACF,CAAC;QAEJ,KAAK,aAAa;YAChB,OAAO;gBACL,IAAI,EAAE,aAAa;gBACnB,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,WAAW,EAAE,WAAW,CAAC,WAAW;gBACpC,KAAK,EAAE,EAAE;gBACT,oBAAoB,EAAE,qCAAqC,CACzD,WAAW,CAAC,IAAI,EAChB,WAAW,CAAC,oBAAoB,CACjC;aACF,CAAC;QAEJ,KAAK,QAAQ;YACX,OAAO;gBACL,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,WAAW,EAAE,WAAW,CAAC,WAAW;gBACpC,KAAK,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC7B,OAAO,EAAE,WAAW,CAAC,OAAO;gBAC5B,oBAAoB,EAAE,qCAAqC,CACzD,WAAW,CAAC,IAAI,EAChB,WAAW,CAAC,oBAAoB,CAC6B;aAChE,CAAC;QAEJ,OAAO,CAAC,CAAC,CAAC;YACR,MAAM,gBAAgB,GAAU,WAAW,CAAC;YAC5C,OAAO,gBAAgB,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,+BAA+B,CAC7C,qBAA4C;IAE5C,OAAO,qBAAqB,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;AACzE,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,uBAAuB,CAAC,IAAmB;IACzD,OAAO;QACL,GAAG,IAAI;QACP,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE;QAC9D,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,6BAA6B,CAAC;KACzE,CAAC;AACJ,CAAC","sourcesContent":["import { v4 as uuidv4 } from 'uuid';\nimport {\n  ExpectedOutcomeField,\n  ExpectedOutcomeSchema,\n  ExpectedOutcomeSchemaField,\n  TestCase,\n  TestCaseInput,\n} from '../../types/llm-test-runner';\nimport { EvaluationApproach } from '../evaluation/constants';\nimport { normalizeEvaluationParametersForField } from '../evaluation/field-evaluation-approach';\n\nexport const DEFAULT_EXPECTED_OUTCOME_SCHEMA: ExpectedOutcomeSchema = [\n  {\n    type: 'textarea',\n    label: 'Expected Outcome',\n    placeholder: 'Enter expected outcome...',\n    rows: 2,\n  },\n];\n\nfunction normalizeExpectedOutcomeField(\n  field: ExpectedOutcomeField,\n): ExpectedOutcomeField {\n  return {\n    ...field,\n    evaluationParameters: normalizeEvaluationParametersForField(\n      field.type,\n      field.evaluationParameters,\n    ),\n  };\n}\n\n/**\n * Creates a new test case with default values\n * @returns A new TestCase object with a unique ID\n */\nexport function createTestCase(\n  expectedOutcomeSchema: ExpectedOutcomeSchema = DEFAULT_EXPECTED_OUTCOME_SCHEMA,\n): TestCase {\n  return {\n    id: uuidv4(),\n    question: '',\n    expectedOutcome: createExpectedOutcomeFromSchema(expectedOutcomeSchema),\n    chatHistory: { enabled: false, value: '' },\n    isRunning: false,\n  };\n}\n\nfunction createExpectedOutcomeFieldFromSchema(\n  schemaField: ExpectedOutcomeSchemaField,\n): ExpectedOutcomeField {\n  switch (schemaField.type) {\n    case 'text':\n      return {\n        type: 'text',\n        label: schemaField.label,\n        placeholder: schemaField.placeholder,\n        value: '',\n        evaluationParameters: normalizeEvaluationParametersForField(\n          schemaField.type,\n          schemaField.evaluationParameters,\n        ),\n      };\n\n    case 'textarea':\n      return {\n        type: 'textarea',\n        label: schemaField.label,\n        placeholder: schemaField.placeholder,\n        rows: schemaField.rows,\n        value: '',\n        evaluationParameters: normalizeEvaluationParametersForField(\n          schemaField.type,\n          schemaField.evaluationParameters,\n        ),\n      };\n\n    case 'chips-input':\n      return {\n        type: 'chips-input',\n        label: schemaField.label,\n        placeholder: schemaField.placeholder,\n        value: [],\n        evaluationParameters: normalizeEvaluationParametersForField(\n          schemaField.type,\n          schemaField.evaluationParameters,\n        ),\n      };\n\n    case 'select':\n      return {\n        type: 'select',\n        label: schemaField.label,\n        placeholder: schemaField.placeholder,\n        value: schemaField.options[0],\n        options: schemaField.options,\n        evaluationParameters: normalizeEvaluationParametersForField(\n          schemaField.type,\n          schemaField.evaluationParameters,\n        ) as { approach: EvaluationApproach.EXACT; threshold?: number },\n      };\n\n    default: {\n      const _exhaustiveCheck: never = schemaField;\n      return _exhaustiveCheck;\n    }\n  }\n}\n\nexport function createExpectedOutcomeFromSchema(\n  expectedOutcomeSchema: ExpectedOutcomeSchema,\n): ExpectedOutcomeField[] {\n  return expectedOutcomeSchema.map(createExpectedOutcomeFieldFromSchema);\n}\n\n/**\n * Creates a runtime test case from validated input data.\n * The input is expected to already satisfy `TestCaseInput`,\n * and this function only performs normalization/defaulting.\n *\n * @param data - Validated test case input\n * @returns A normalized TestCase object with runtime defaults applied\n */\nexport function createTestCaseFromInput(data: TestCaseInput): TestCase {\n  return {\n    ...data,\n    chatHistory: data.chatHistory ?? { enabled: false, value: '' },\n    expectedOutcome: data.expectedOutcome.map(normalizeExpectedOutcomeField),\n  };\n}\n"]}
+{"version":3,"file":"test-case-factory.js","sourceRoot":"","sources":["../../../src/lib/test-cases/test-case-factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AASpC,OAAO,EAAE,qCAAqC,EAAE,MAAM,yCAAyC,CAAC;AAEhG,MAAM,CAAC,MAAM,+BAA+B,GAA0B;IACpE;QACE,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,kBAAkB;QACzB,WAAW,EAAE,2BAA2B;QACxC,IAAI,EAAE,CAAC;KACR;CACF,CAAC;AAEF,SAAS,6BAA6B,CACpC,KAA2B;IAE3B,OAAO;QACL,GAAG,KAAK;QACR,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE;QAC5D,oBAAoB,EAAE,qCAAqC,CACzD,KAAK,CAAC,IAAI,EACV,KAAK,CAAC,oBAAoB,CAC3B;KACF,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAC5B,wBAA+C,+BAA+B;IAE9E,OAAO;QACL,EAAE,EAAE,MAAM,EAAE;QACZ,QAAQ,EAAE,EAAE;QACZ,eAAe,EAAE,+BAA+B,CAAC,qBAAqB,CAAC;QACvE,WAAW,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE;QAC1C,SAAS,EAAE,KAAK;KACjB,CAAC;AACJ,CAAC;AAED,SAAS,oCAAoC,CAC3C,WAAuC;IAEvC,QAAQ,WAAW,CAAC,IAAI,EAAE,CAAC;QACzB,KAAK,MAAM;YACT,OAAO;gBACL,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,WAAW,EAAE,WAAW,CAAC,WAAW;gBACpC,gBAAgB,EAAE,WAAW,CAAC,gBAAgB,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE;gBAClE,KAAK,EAAE,EAAE;gBACT,oBAAoB,EAAE,qCAAqC,CACzD,WAAW,CAAC,IAAI,EAChB,WAAW,CAAC,oBAAoB,CACjC;aACF,CAAC;QAEJ,KAAK,UAAU;YACb,OAAO;gBACL,IAAI,EAAE,UAAU;gBAChB,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,WAAW,EAAE,WAAW,CAAC,WAAW;gBACpC,gBAAgB,EAAE,WAAW,CAAC,gBAAgB,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE;gBAClE,IAAI,EAAE,WAAW,CAAC,IAAI;gBACtB,KAAK,EAAE,EAAE;gBACT,oBAAoB,EAAE,qCAAqC,CACzD,WAAW,CAAC,IAAI,EAChB,WAAW,CAAC,oBAAoB,CACjC;aACF,CAAC;QAEJ,KAAK,aAAa;YAChB,OAAO;gBACL,IAAI,EAAE,aAAa;gBACnB,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,WAAW,EAAE,WAAW,CAAC,WAAW;gBACpC,gBAAgB,EAAE,WAAW,CAAC,gBAAgB,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE;gBAClE,KAAK,EAAE,EAAE;gBACT,oBAAoB,EAAE,qCAAqC,CACzD,WAAW,CAAC,IAAI,EAChB,WAAW,CAAC,oBAAoB,CACjC;aACF,CAAC;QAEJ,KAAK,QAAQ;YACX,OAAO;gBACL,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,WAAW,EAAE,WAAW,CAAC,WAAW;gBACpC,gBAAgB,EAAE,WAAW,CAAC,gBAAgB,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE;gBAClE,KAAK,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC7B,OAAO,EAAE,WAAW,CAAC,OAAO;gBAC5B,oBAAoB,EAAE,qCAAqC,CACzD,WAAW,CAAC,IAAI,EAChB,WAAW,CAAC,oBAAoB,CAC6B;aAChE,CAAC;QAEJ,OAAO,CAAC,CAAC,CAAC;YACR,MAAM,gBAAgB,GAAU,WAAW,CAAC;YAC5C,OAAO,gBAAgB,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,+BAA+B,CAC7C,qBAA4C;IAE5C,OAAO,qBAAqB,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;AACzE,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,uBAAuB,CAAC,IAAmB;IACzD,OAAO;QACL,GAAG,IAAI;QACP,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE;QAC9D,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,6BAA6B,CAAC;KACzE,CAAC;AACJ,CAAC","sourcesContent":["import { v4 as uuidv4 } from 'uuid';\nimport {\n  ExpectedOutcomeField,\n  ExpectedOutcomeSchema,\n  ExpectedOutcomeSchemaField,\n  TestCase,\n  TestCaseInput,\n} from '../../types/llm-test-runner';\nimport { EvaluationApproach } from '../evaluation/constants';\nimport { normalizeEvaluationParametersForField } from '../evaluation/field-evaluation-approach';\n\nexport const DEFAULT_EXPECTED_OUTCOME_SCHEMA: ExpectedOutcomeSchema = [\n  {\n    type: 'textarea',\n    label: 'Expected Outcome',\n    placeholder: 'Enter expected outcome...',\n    rows: 2,\n  },\n];\n\nfunction normalizeExpectedOutcomeField(\n  field: ExpectedOutcomeField,\n): ExpectedOutcomeField {\n  return {\n    ...field,\n    evaluationSource: field.evaluationSource || { type: 'text' },\n    evaluationParameters: normalizeEvaluationParametersForField(\n      field.type,\n      field.evaluationParameters,\n    ),\n  };\n}\n\n/**\n * Creates a new test case with default values\n * @returns A new TestCase object with a unique ID\n */\nexport function createTestCase(\n  expectedOutcomeSchema: ExpectedOutcomeSchema = DEFAULT_EXPECTED_OUTCOME_SCHEMA,\n): TestCase {\n  return {\n    id: uuidv4(),\n    question: '',\n    expectedOutcome: createExpectedOutcomeFromSchema(expectedOutcomeSchema),\n    chatHistory: { enabled: false, value: '' },\n    isRunning: false,\n  };\n}\n\nfunction createExpectedOutcomeFieldFromSchema(\n  schemaField: ExpectedOutcomeSchemaField,\n): ExpectedOutcomeField {\n  switch (schemaField.type) {\n    case 'text':\n      return {\n        type: 'text',\n        label: schemaField.label,\n        placeholder: schemaField.placeholder,\n        evaluationSource: schemaField.evaluationSource || { type: 'text' },\n        value: '',\n        evaluationParameters: normalizeEvaluationParametersForField(\n          schemaField.type,\n          schemaField.evaluationParameters,\n        ),\n      };\n\n    case 'textarea':\n      return {\n        type: 'textarea',\n        label: schemaField.label,\n        placeholder: schemaField.placeholder,\n        evaluationSource: schemaField.evaluationSource || { type: 'text' },\n        rows: schemaField.rows,\n        value: '',\n        evaluationParameters: normalizeEvaluationParametersForField(\n          schemaField.type,\n          schemaField.evaluationParameters,\n        ),\n      };\n\n    case 'chips-input':\n      return {\n        type: 'chips-input',\n        label: schemaField.label,\n        placeholder: schemaField.placeholder,\n        evaluationSource: schemaField.evaluationSource || { type: 'text' },\n        value: [],\n        evaluationParameters: normalizeEvaluationParametersForField(\n          schemaField.type,\n          schemaField.evaluationParameters,\n        ),\n      };\n\n    case 'select':\n      return {\n        type: 'select',\n        label: schemaField.label,\n        placeholder: schemaField.placeholder,\n        evaluationSource: schemaField.evaluationSource || { type: 'text' },\n        value: schemaField.options[0],\n        options: schemaField.options,\n        evaluationParameters: normalizeEvaluationParametersForField(\n          schemaField.type,\n          schemaField.evaluationParameters,\n        ) as { approach: EvaluationApproach.EXACT; threshold?: number },\n      };\n\n    default: {\n      const _exhaustiveCheck: never = schemaField;\n      return _exhaustiveCheck;\n    }\n  }\n}\n\nexport function createExpectedOutcomeFromSchema(\n  expectedOutcomeSchema: ExpectedOutcomeSchema,\n): ExpectedOutcomeField[] {\n  return expectedOutcomeSchema.map(createExpectedOutcomeFieldFromSchema);\n}\n\n/**\n * Creates a runtime test case from validated input data.\n * The input is expected to already satisfy `TestCaseInput`,\n * and this function only performs normalization/defaulting.\n *\n * @param data - Validated test case input\n * @returns A normalized TestCase object with runtime defaults applied\n */\nexport function createTestCaseFromInput(data: TestCaseInput): TestCase {\n  return {\n    ...data,\n    chatHistory: data.chatHistory ?? { enabled: false, value: '' },\n    expectedOutcome: data.expectedOutcome.map(normalizeExpectedOutcomeField),\n  };\n}\n"]}

package/dist/collection/lib/test-cases/test-case-mutations.js

@@ -1,4 +1,13 @@
 import { normalizeEvaluationParametersForField } from "../evaluation/field-evaluation-approach";
+function isChipsInputField(field) {
+    return field.type === 'chips-input';
+}
+function isTextareaField(field) {
+    return field.type === 'textarea';
+}
+function isDynamicTextareaField(field) {
+    return isTextareaField(field) && field.outcomeMode === 'dynamic';
+}
 export function applyExpectedOutcomeChange(testCase, change) {
     const { index } = change;
     const expectedOutcome = [...(testCase.expectedOutcome || [])];
@@ -6,73 +15,99 @@ export function applyExpectedOutcomeChange(testCase, change) {
     if (!target) {
         return testCase;
     }
+    const commit = (updatedField) => {
+        expectedOutcome[index] = updatedField;
+        return { ...testCase, expectedOutcome };
+    };
     switch (change.operation) {
         case 'set-value': {
-            if (target.type === 'chips-input') {
+            if (isChipsInputField(target)) {
                 return testCase;
             }
-            if (target.type === 'textarea' && target.outcomeMode === 'dynamic') {
+            if (isDynamicTextareaField(target)) {
                 return testCase;
             }
-            expectedOutcome[index] = {
+            return commit({
                 ...target,
                 value: change.value,
-            };
-            return { ...testCase, expectedOutcome };
+            });
         }
         case 'add-chip': {
-            if (target.type !== 'chips-input') {
+            if (!isChipsInputField(target)) {
                 return testCase;
             }
-            expectedOutcome[index] = {
+            return commit({
                 ...target,
                 value: [...target.value, change.value],
-            };
-            return { ...testCase, expectedOutcome };
+            });
         }
         case 'remove-chip': {
-            if (target.type !== 'chips-input') {
+            if (!isChipsInputField(target)) {
                 return testCase;
             }
-            expectedOutcome[index] = {
+            return commit({
                 ...target,
                 value: target.value.filter(chip => chip !== change.value),
-            };
-            return { ...testCase, expectedOutcome };
+            });
         }
         case 'set-evaluation-approach':
             return updateExpectedOutcomeFieldApproach(testCase, index, change.value);
         case 'set-outcome-mode': {
-            if (target.type !== 'textarea') {
+            if (!isTextareaField(target)) {
                 return testCase;
             }
             const mode = change.value;
             if (mode === 'static') {
                 const { resolutionQuery: _, ...rest } = target;
-                expectedOutcome[index] = {
+                return commit({
                     ...rest,
                     outcomeMode: 'static',
                     value: '',
-                };
+                });
             }
             else {
-                expectedOutcome[index] = {
+                return commit({
                     ...target,
                     outcomeMode: 'dynamic',
                     value: '',
-                };
+                });
             }
-            return { ...testCase, expectedOutcome };
         }
         case 'set-resolution-query': {
-            if (target.type !== 'textarea' || target.outcomeMode !== 'dynamic') {
+            if (!isDynamicTextareaField(target)) {
                 return testCase;
             }
-            expectedOutcome[index] = {
+            return commit({
                 ...target,
                 resolutionQuery: change.value,
-            };
-            return { ...testCase, expectedOutcome };
+            });
+        }
+        case 'set-evaluation-source-type': {
+            if (change.value === 'text') {
+                return commit({
+                    ...target,
+                    evaluationSource: { type: 'text' },
+                });
+            }
+            const extractorId = target.evaluationSource?.type === 'custom'
+                ? target.evaluationSource.extractorId
+                : (change.fallbackExtractorId ?? '');
+            return commit({
+                ...target,
+                evaluationSource: {
+                    type: 'custom',
+                    extractorId,
+                },
+            });
+        }
+        case 'set-evaluation-source-extractor': {
+            return commit({
+                ...target,
+                evaluationSource: {
+                    type: 'custom',
+                    extractorId: change.value,
+                },
+            });
         }
     }
 }

package/dist/collection/lib/test-cases/test-case-mutations.js.map

@@ -1 +1 @@
-{"version":3,"file":"test-case-mutations.js","sourceRoot":"","sources":["../../../src/lib/test-cases/test-case-mutations.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,qCAAqC,EAAE,MAAM,yCAAyC,CAAC;AAkChG,MAAM,UAAU,0BAA0B,CACxC,QAAkB,EAClB,MAA6B;IAE7B,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IACzB,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,CAAC;IAC9D,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IAEtC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,QAAQ,MAAM,CAAC,SAAS,EAAE,CAAC;QACzB,KAAK,WAAW,CAAC,CAAC,CAAC;YACjB,IAAI,MAAM,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;gBAClC,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,IAAI,MAAM,CAAC,IAAI,KAAK,UAAU,IAAI,MAAM,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;gBACnE,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,eAAe,CAAC,KAAK,CAAC,GAAG;gBACvB,GAAG,MAAM;gBACT,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC;YACF,OAAO,EAAE,GAAG,QAAQ,EAAE,eAAe,EAAE,CAAC;QAC1C,CAAC;QACD,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,IAAI,MAAM,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;gBAClC,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,eAAe,CAAC,KAAK,CAAC,GAAG;gBACvB,GAAG,MAAM;gBACT,KAAK,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;aACvC,CAAC;YACF,OAAO,EAAE,GAAG,QAAQ,EAAE,eAAe,EAAE,CAAC;QAC1C,CAAC;QACD,KAAK,aAAa,CAAC,CAAC,CAAC;YACnB,IAAI,MAAM,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;gBAClC,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,eAAe,CAAC,KAAK,CAAC,GAAG;gBACvB,GAAG,MAAM;gBACT,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,MAAM,CAAC,KAAK,CAAC;aAC1D,CAAC;YACF,OAAO,EAAE,GAAG,QAAQ,EAAE,eAAe,EAAE,CAAC;QAC1C,CAAC;QACD,KAAK,yBAAyB;YAC5B,OAAO,kCAAkC,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3E,KAAK,kBAAkB,CAAC,CAAC,CAAC;YACxB,IAAI,MAAM,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;gBAC/B,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC;YAC1B,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACtB,MAAM,EAAE,eAAe,EAAE,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC;gBAC/C,eAAe,CAAC,KAAK,CAAC,GAAG;oBACvB,GAAG,IAAI;oBACP,WAAW,EAAE,QAAQ;oBACrB,KAAK,EAAE,EAAE;iBACV,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,eAAe,CAAC,KAAK,CAAC,GAAG;oBACvB,GAAG,MAAM;oBACT,WAAW,EAAE,SAAS;oBACtB,KAAK,EAAE,EAAE;iBACV,CAAC;YACJ,CAAC;YACD,OAAO,EAAE,GAAG,QAAQ,EAAE,eAAe,EAAE,CAAC;QAC1C,CAAC;QACD,KAAK,sBAAsB,CAAC,CAAC,CAAC;YAC5B,IAAI,MAAM,CAAC,IAAI,KAAK,UAAU,IAAI,MAAM,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;gBACnE,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,eAAe,CAAC,KAAK,CAAC,GAAG;gBACvB,GAAG,MAAM;gBACT,eAAe,EAAE,MAAM,CAAC,KAAK;aAC9B,CAAC;YACF,OAAO,EAAE,GAAG,QAAQ,EAAE,eAAe,EAAE,CAAC;QAC1C,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kCAAkC,CAChD,QAAkB,EAClB,UAAkB,EAClB,QAA4B;IAE5B,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,CAAC;IAC9D,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,CAAC,CAAC;IAE3C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,MAAM,2BAA2B,GAAG,MAAM,CAAC,oBAAoB,CAAC;IAChE,eAAe,CAAC,UAAU,CAAC,GAAG;QAC5B,GAAG,MAAM;QACT,oBAAoB,EAAE,qCAAqC,CAAC,MAAM,CAAC,IAAI,EAAE;YACvE,GAAG,2BAA2B;YAC9B,QAAQ;SACT,CAAC;KACH,CAAC;IAEF,OAAO;QACL,GAAG,QAAQ;QACX,eAAe;KAChB,CAAC;AACJ,CAAC","sourcesContent":["import {\n  TestCase,\n  type ExpectedOutcomeMode,\n} from '../../types/llm-test-runner';\nimport { EvaluationApproach } from '../evaluation/constants';\nimport { normalizeEvaluationParametersForField } from '../evaluation/field-evaluation-approach';\n\nexport type ExpectedOutcomeChange =\n  | {\n      index: number;\n      operation: 'set-value';\n      value: string;\n    }\n  | {\n      index: number;\n      operation: 'add-chip';\n      value: string;\n    }\n  | {\n      index: number;\n      operation: 'remove-chip';\n      value: string;\n    }\n  | {\n      index: number;\n      operation: 'set-evaluation-approach';\n      value: EvaluationApproach;\n    }\n  | {\n      index: number;\n      operation: 'set-outcome-mode';\n      value: ExpectedOutcomeMode;\n    }\n  | {\n      index: number;\n      operation: 'set-resolution-query';\n      value: string;\n    };\n\nexport function applyExpectedOutcomeChange(\n  testCase: TestCase,\n  change: ExpectedOutcomeChange,\n): TestCase {\n  const { index } = change;\n  const expectedOutcome = [...(testCase.expectedOutcome || [])];\n  const target = expectedOutcome[index];\n\n  if (!target) {\n    return testCase;\n  }\n\n  switch (change.operation) {\n    case 'set-value': {\n      if (target.type === 'chips-input') {\n        return testCase;\n      }\n      if (target.type === 'textarea' && target.outcomeMode === 'dynamic') {\n        return testCase;\n      }\n      expectedOutcome[index] = {\n        ...target,\n        value: change.value,\n      };\n      return { ...testCase, expectedOutcome };\n    }\n    case 'add-chip': {\n      if (target.type !== 'chips-input') {\n        return testCase;\n      }\n      expectedOutcome[index] = {\n        ...target,\n        value: [...target.value, change.value],\n      };\n      return { ...testCase, expectedOutcome };\n    }\n    case 'remove-chip': {\n      if (target.type !== 'chips-input') {\n        return testCase;\n      }\n      expectedOutcome[index] = {\n        ...target,\n        value: target.value.filter(chip => chip !== change.value),\n      };\n      return { ...testCase, expectedOutcome };\n    }\n    case 'set-evaluation-approach':\n      return updateExpectedOutcomeFieldApproach(testCase, index, change.value);\n    case 'set-outcome-mode': {\n      if (target.type !== 'textarea') {\n        return testCase;\n      }\n      const mode = change.value;\n      if (mode === 'static') {\n        const { resolutionQuery: _, ...rest } = target;\n        expectedOutcome[index] = {\n          ...rest,\n          outcomeMode: 'static',\n          value: '',\n        };\n      } else {\n        expectedOutcome[index] = {\n          ...target,\n          outcomeMode: 'dynamic',\n          value: '',\n        };\n      }\n      return { ...testCase, expectedOutcome };\n    }\n    case 'set-resolution-query': {\n      if (target.type !== 'textarea' || target.outcomeMode !== 'dynamic') {\n        return testCase;\n      }\n      expectedOutcome[index] = {\n        ...target,\n        resolutionQuery: change.value,\n      };\n      return { ...testCase, expectedOutcome };\n    }\n  }\n}\n\n/**\n * Updates the evaluation approach for a specific expected outcome field.\n * Select fields always use exact matching.\n */\nexport function updateExpectedOutcomeFieldApproach(\n  testCase: TestCase,\n  fieldIndex: number,\n  approach: EvaluationApproach,\n): TestCase {\n  const expectedOutcome = [...(testCase.expectedOutcome || [])];\n  const target = expectedOutcome[fieldIndex];\n\n  if (!target) {\n    return testCase;\n  }\n\n  const currentEvaluationParameters = target.evaluationParameters;\n  expectedOutcome[fieldIndex] = {\n    ...target,\n    evaluationParameters: normalizeEvaluationParametersForField(target.type, {\n      ...currentEvaluationParameters,\n      approach,\n    }),\n  };\n\n  return {\n    ...testCase,\n    expectedOutcome,\n  };\n}\n"]}
+{"version":3,"file":"test-case-mutations.js","sourceRoot":"","sources":["../../../src/lib/test-cases/test-case-mutations.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,qCAAqC,EAAE,MAAM,yCAAyC,CAAC;AAEhG,SAAS,iBAAiB,CACxB,KAA2B;IAE3B,OAAO,KAAK,CAAC,IAAI,KAAK,aAAa,CAAC;AACtC,CAAC;AAED,SAAS,eAAe,CACtB,KAA2B;IAE3B,OAAO,KAAK,CAAC,IAAI,KAAK,UAAU,CAAC;AACnC,CAAC;AAED,SAAS,sBAAsB,CAC7B,KAA2B;IAE3B,OAAO,eAAe,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC;AACnE,CAAC;AA8CD,MAAM,UAAU,0BAA0B,CACxC,QAAkB,EAClB,MAA6B;IAE7B,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IACzB,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,CAAC;IAC9D,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IAEtC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,CAAC,YAAkC,EAAY,EAAE;QAC9D,eAAe,CAAC,KAAK,CAAC,GAAG,YAAY,CAAC;QACtC,OAAO,EAAE,GAAG,QAAQ,EAAE,eAAe,EAAE,CAAC;IAC1C,CAAC,CAAC;IAEF,QAAQ,MAAM,CAAC,SAAS,EAAE,CAAC;QACzB,KAAK,WAAW,CAAC,CAAC,CAAC;YACjB,IAAI,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9B,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,IAAI,sBAAsB,CAAC,MAAM,CAAC,EAAE,CAAC;gBACnC,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,OAAO,MAAM,CAAC;gBACZ,GAAG,MAAM;gBACT,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC,CAAC;QACL,CAAC;QACD,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC/B,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,OAAO,MAAM,CAAC;gBACZ,GAAG,MAAM;gBACT,KAAK,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;aACvC,CAAC,CAAC;QACL,CAAC;QACD,KAAK,aAAa,CAAC,CAAC,CAAC;YACnB,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC/B,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,OAAO,MAAM,CAAC;gBACZ,GAAG,MAAM;gBACT,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,MAAM,CAAC,KAAK,CAAC;aAC1D,CAAC,CAAC;QACL,CAAC;QACD,KAAK,yBAAyB;YAC5B,OAAO,kCAAkC,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3E,KAAK,kBAAkB,CAAC,CAAC,CAAC;YACxB,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7B,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC;YAC1B,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACtB,MAAM,EAAE,eAAe,EAAE,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC;gBAC/C,OAAO,MAAM,CAAC;oBACZ,GAAG,IAAI;oBACP,WAAW,EAAE,QAAQ;oBACrB,KAAK,EAAE,EAAE;iBACV,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,OAAO,MAAM,CAAC;oBACZ,GAAG,MAAM;oBACT,WAAW,EAAE,SAAS;oBACtB,KAAK,EAAE,EAAE;iBACV,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,KAAK,sBAAsB,CAAC,CAAC,CAAC;YAC5B,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,EAAE,CAAC;gBACpC,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,OAAO,MAAM,CAAC;gBACZ,GAAG,MAAM;gBACT,eAAe,EAAE,MAAM,CAAC,KAAK;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,KAAK,4BAA4B,CAAC,CAAC,CAAC;YAClC,IAAI,MAAM,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;gBAC5B,OAAO,MAAM,CAAC;oBACZ,GAAG,MAAM;oBACT,gBAAgB,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;iBACnC,CAAC,CAAC;YACL,CAAC;YAED,MAAM,WAAW,GACf,MAAM,CAAC,gBAAgB,EAAE,IAAI,KAAK,QAAQ;gBACxC,CAAC,CAAC,MAAM,CAAC,gBAAgB,CAAC,WAAW;gBACrC,CAAC,CAAC,CAAC,MAAM,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC;YACzC,OAAO,MAAM,CAAC;gBACZ,GAAG,MAAM;gBACT,gBAAgB,EAAE;oBAChB,IAAI,EAAE,QAAQ;oBACd,WAAW;iBACZ;aACF,CAAC,CAAC;QACL,CAAC;QACD,KAAK,iCAAiC,CAAC,CAAC,CAAC;YACvC,OAAO,MAAM,CAAC;gBACZ,GAAG,MAAM;gBACT,gBAAgB,EAAE;oBAChB,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,MAAM,CAAC,KAAK;iBAC1B;aACF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kCAAkC,CAChD,QAAkB,EAClB,UAAkB,EAClB,QAA4B;IAE5B,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,CAAC;IAC9D,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,CAAC,CAAC;IAE3C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,MAAM,2BAA2B,GAAG,MAAM,CAAC,oBAAoB,CAAC;IAChE,eAAe,CAAC,UAAU,CAAC,GAAG;QAC5B,GAAG,MAAM;QACT,oBAAoB,EAAE,qCAAqC,CAAC,MAAM,CAAC,IAAI,EAAE;YACvE,GAAG,2BAA2B;YAC9B,QAAQ;SACT,CAAC;KACH,CAAC;IAEF,OAAO;QACL,GAAG,QAAQ;QACX,eAAe;KAChB,CAAC;AACJ,CAAC","sourcesContent":["import {\n  TestCase,\n  type ExpectedOutcomeField,\n  type EvaluationSource,\n  type ExpectedOutcomeMode,\n} from '../../types/llm-test-runner';\nimport { EvaluationApproach } from '../evaluation/constants';\nimport { normalizeEvaluationParametersForField } from '../evaluation/field-evaluation-approach';\n\nfunction isChipsInputField(\n  field: ExpectedOutcomeField,\n): field is Extract<ExpectedOutcomeField, { type: 'chips-input' }> {\n  return field.type === 'chips-input';\n}\n\nfunction isTextareaField(\n  field: ExpectedOutcomeField,\n): field is Extract<ExpectedOutcomeField, { type: 'textarea' }> {\n  return field.type === 'textarea';\n}\n\nfunction isDynamicTextareaField(\n  field: ExpectedOutcomeField,\n): field is Extract<ExpectedOutcomeField, { type: 'textarea' }> {\n  return isTextareaField(field) && field.outcomeMode === 'dynamic';\n}\n\n\nexport type ExpectedOutcomeChange =\n  | {\n      index: number;\n      operation: 'set-value';\n      value: string;\n    }\n  | {\n      index: number;\n      operation: 'add-chip';\n      value: string;\n    }\n  | {\n      index: number;\n      operation: 'remove-chip';\n      value: string;\n    }\n  | {\n      index: number;\n      operation: 'set-evaluation-approach';\n      value: EvaluationApproach;\n    }\n  | {\n      index: number;\n      operation: 'set-outcome-mode';\n      value: ExpectedOutcomeMode;\n    }\n  | {\n      index: number;\n      operation: 'set-resolution-query';\n      value: string;\n    }\n  | {\n      index: number;\n      operation: 'set-evaluation-source-type';\n      value: EvaluationSource['type'];\n      fallbackExtractorId?: string;\n    }\n  | {\n      index: number;\n      operation: 'set-evaluation-source-extractor';\n      value: string;\n    };\n\nexport function applyExpectedOutcomeChange(\n  testCase: TestCase,\n  change: ExpectedOutcomeChange,\n): TestCase {\n  const { index } = change;\n  const expectedOutcome = [...(testCase.expectedOutcome || [])];\n  const target = expectedOutcome[index];\n\n  if (!target) {\n    return testCase;\n  }\n\n  const commit = (updatedField: ExpectedOutcomeField): TestCase => {\n    expectedOutcome[index] = updatedField;\n    return { ...testCase, expectedOutcome };\n  };\n\n  switch (change.operation) {\n    case 'set-value': {\n      if (isChipsInputField(target)) {\n        return testCase;\n      }\n      if (isDynamicTextareaField(target)) {\n        return testCase;\n      }\n      return commit({\n        ...target,\n        value: change.value,\n      });\n    }\n    case 'add-chip': {\n      if (!isChipsInputField(target)) {\n        return testCase;\n      }\n      return commit({\n        ...target,\n        value: [...target.value, change.value],\n      });\n    }\n    case 'remove-chip': {\n      if (!isChipsInputField(target)) {\n        return testCase;\n      }\n      return commit({\n        ...target,\n        value: target.value.filter(chip => chip !== change.value),\n      });\n    }\n    case 'set-evaluation-approach':\n      return updateExpectedOutcomeFieldApproach(testCase, index, change.value);\n    case 'set-outcome-mode': {\n      if (!isTextareaField(target)) {\n        return testCase;\n      }\n      const mode = change.value;\n      if (mode === 'static') {\n        const { resolutionQuery: _, ...rest } = target;\n        return commit({\n          ...rest,\n          outcomeMode: 'static',\n          value: '',\n        });\n      } else {\n        return commit({\n          ...target,\n          outcomeMode: 'dynamic',\n          value: '',\n        });\n      }\n    }\n    case 'set-resolution-query': {\n      if (!isDynamicTextareaField(target)) {\n        return testCase;\n      }\n      return commit({\n        ...target,\n        resolutionQuery: change.value,\n      });\n    }\n    case 'set-evaluation-source-type': {\n      if (change.value === 'text') {\n        return commit({\n          ...target,\n          evaluationSource: { type: 'text' },\n        });\n      }\n\n      const extractorId =\n        target.evaluationSource?.type === 'custom'\n          ? target.evaluationSource.extractorId\n          : (change.fallbackExtractorId ?? '');\n      return commit({\n        ...target,\n        evaluationSource: {\n          type: 'custom',\n          extractorId,\n        },\n      });\n    }\n    case 'set-evaluation-source-extractor': {\n      return commit({\n        ...target,\n        evaluationSource: {\n          type: 'custom',\n          extractorId: change.value,\n        },\n      });\n    }\n  }\n}\n\n/**\n * Updates the evaluation approach for a specific expected outcome field.\n * Select fields always use exact matching.\n */\nexport function updateExpectedOutcomeFieldApproach(\n  testCase: TestCase,\n  fieldIndex: number,\n  approach: EvaluationApproach,\n): TestCase {\n  const expectedOutcome = [...(testCase.expectedOutcome || [])];\n  const target = expectedOutcome[fieldIndex];\n\n  if (!target) {\n    return testCase;\n  }\n\n  const currentEvaluationParameters = target.evaluationParameters;\n  expectedOutcome[fieldIndex] = {\n    ...target,\n    evaluationParameters: normalizeEvaluationParametersForField(target.type, {\n      ...currentEvaluationParameters,\n      approach,\n    }),\n  };\n\n  return {\n    ...testCase,\n    expectedOutcome,\n  };\n}\n"]}