npm - llm-testrunner-components - Versions diffs - 1.2.4 → 1.3.1 - Mend

llm-testrunner-components 1.2.4 → 1.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (93) hide show

package/dist/collection/components/llm-test-runner/test-cases/llm-test-case-row.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"llm-test-case-row.js","sourceRoot":"","sources":["../../../../src/components/llm-test-runner/test-cases/llm-test-case-row.tsx"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAuB,MAAM,eAAe,CAAC;AAEvD,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AACpE,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAkB,MAAM,0BAA0B,CAAC;AACzE,OAAO,EAEL,uBAAuB,GACxB,MAAM,6BAA6B,CAAC;~~AAqBrC~~,MAAM,CAAC,MAAM,cAAc,GAA6C,CAAC,EACvE,QAAQ,EACR,0BAA0B,GAAG,KAAK,EAClC,KAAK,EACL,QAAQ,EACR,oBAAoB,EACpB,uBAAuB,EACvB,mBAAmB,GACpB,EAAE,EAAE;IACH,MAAM,cAAc,GAAmB;QACrC,IAAI,EAAE,UAAU;QAChB,SAAS,EAAE,aAAa,CAAC,SAAS;QAClC,IAAI,EAAE,MAAM;QACZ,KAAK,EAAE,UAAU;QACjB,WAAW,EAAE,6BAA6B;QAC1C,QAAQ,EAAE,IAAI;QACd,IAAI,EAAE,CAAC;KACR,CAAC;IACF,OAAO,CACL,WAAK,KAAK,EAAC,eAAe,EAAC,GAAG,EAAE,QAAQ,CAAC,EAAE;QACzC,WAAK,KAAK,EAAC,6BAA6B;YACtC,oBACE,MAAM,EAAE,cAAc,EACtB,KAAK,EAAE,QAAQ,CAAC,QAAQ,EACxB,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CACnB,oBAAoB,CAAC;oBACnB,MAAM,EAAE;wBACN,UAAU,EAAE,QAAQ,CAAC,EAAE;wBACvB,GAAG,EAAE,UAAU;wBACf,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK;qBACtB;iBACiE,CAAC,GAEvE;YACF,oBACE,kBAAkB,EAAE,QAAQ,CAAC,WAAW,EAAE,OAAO,IAAI,KAAK,EAC1D,gBAAgB,EAAE,QAAQ,CAAC,WAAW,EAAE,KAAK,IAAI,EAAE,EACnD,mBAAmB,EAAE,CAAC,CAAQ,EAAE,EAAE;oBAChC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAI,CAA0C;yBACnE,MAAM,CAAC;oBACV,mBAAmB,CAAC;wBAClB,MAAM,EAAE;4BACN,UAAU,EAAE,QAAQ,CAAC,EAAE;4BACvB,OAAO;4BACP,KAAK;yBACN;qBACyC,CAAC,CAAC;gBAChD,CAAC,GACD;YACF,EAAC,uBAAuB,IACtB,UAAU,EAAE,QAAQ,CAAC,EAAE,EACvB,MAAM,EAAE,QAAQ,CAAC,eAAe,IAAI,EAAE,EACtC,0BAA0B,EAAE,0BAA0B,EACtD,uBAAuB,EAAE,uBAAuB,GAChD,CACE;QAEN,EAAC,cAAc,IAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,GAAI;QAE1E,EAAC,iBAAiB,IAChB,MAAM,EAAE,QAAQ,CAAC,gBAAgB,EACjC,SAAS,EAAE,QAAQ,CAAC,SAAS,GAC7B;QAEF,EAAC,UAAU,IACT,SAAS,EAAE,QAAQ,CAAC,SAAS,EAC7B,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,EAClC,KAAK,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,EAC5B,QAAQ,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,GACrC,CACE,CACP,CAAC;AACJ,CAAC,CAAC","sourcesContent":["import { h, FunctionalComponent } from '@stencil/core';\nimport { TestCase } from '../../../types/llm-test-runner';\nimport { ResponseOutput } from './output/response-output';\nimport { EvaluationSummary } from './evaluation/evaluation-summary';\nimport { RowActions } from './actions/row-actions';\nimport { FormFieldType, TextAreaConfig } from '../../../lib/form/schema';\nimport {\n ExpectedOutcomeChangeDetail,\n ExpectedOutcomeRenderer,\n} from './expected-outcome-renderer';\nimport type { ChatHistoryChangeDetail } from './chat-history';\n\nexport type ChatHistoryRowChangeDetail = {\n testCaseId: string;\n} & ChatHistoryChangeDetail;\n\nexport interface LLMTestCaseRowProps {\n testCase: TestCase;\n dynamicResolutionSupported?: boolean;\n onRun: (testCase: TestCase) => void;\n onDelete: (id: string) => void;\n handleTestCaseChange: (\n e: CustomEvent<{ testCaseId: string; key: string; value: string }>,\n ) => void;\n onExpectedOutcomeChange: (\n e: CustomEvent<ExpectedOutcomeChangeDetail>,\n ) => void;\n onChatHistoryChange: (e: CustomEvent<ChatHistoryRowChangeDetail>) => void;\n}\n\nexport const LLMTestCaseRow: FunctionalComponent<LLMTestCaseRowProps> = ({\n testCase,\n dynamicResolutionSupported = false,\n onRun,\n onDelete,\n handleTestCaseChange,\n onExpectedOutcomeChange,\n onChatHistoryChange,\n}) => {\n const questionConfig: TextAreaConfig = {\n name: 'question',\n fieldType: FormFieldType.TEXT_AREA,\n type: 'text',\n label: 'Question',\n placeholder: 'Enter your question here...',\n required: true,\n rows: 3,\n };\n return (\n <div class=\"test-case-row\" key={testCase.id}>\n <div class=\"test-case-row__input-column\">\n <app-textarea\n config={questionConfig}\n value={testCase.question}\n onValueChange={(e) =>\n handleTestCaseChange({\n detail: {\n testCaseId: testCase.id,\n key: 'question',\n value: e.detail.value,\n },\n } as CustomEvent<{ testCaseId: string; key: string; value: string }>)\n }\n />\n <chat-history\n chatHistoryEnabled={testCase.chatHistory?.enabled ?? false}\n chatHistoryValue={testCase.chatHistory?.value ?? ''}\n onChatHistoryChange={(e: Event) => {\n const { enabled, value } = (e as CustomEvent<ChatHistoryChangeDetail>)\n .detail;\n onChatHistoryChange({\n detail: {\n testCaseId: testCase.id,\n enabled,\n value,\n },\n } as CustomEvent<ChatHistoryRowChangeDetail>);\n }}\n />\n <ExpectedOutcomeRenderer\n testCaseId={testCase.id}\n fields={testCase.expectedOutcome \|\| []}\n dynamicResolutionSupported={dynamicResolutionSupported}\n onExpectedOutcomeChange={onExpectedOutcomeChange}\n />\n </div>\n\n <ResponseOutput output={testCase.output} isRunning={testCase.isRunning} />\n\n <EvaluationSummary\n result={testCase.evaluationResult}\n isRunning={testCase.isRunning}\n />\n\n <RowActions\n isRunning={testCase.isRunning}\n canRun={!!testCase.question.trim()}\n onRun={() => onRun(testCase)}\n onDelete={() => onDelete(testCase.id)}\n />\n </div>\n );\n};\n"]}
1	+ {"version":3,"file":"llm-test-case-row.js","sourceRoot":"","sources":["../../../../src/components/llm-test-runner/test-cases/llm-test-case-row.tsx"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAuB,MAAM,eAAe,CAAC;AAEvD,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AACpE,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAkB,MAAM,0BAA0B,CAAC;AACzE,OAAO,EAEL,uBAAuB,GACxB,MAAM,6BAA6B,CAAC;AAsBrC,MAAM,CAAC,MAAM,cAAc,GAA6C,CAAC,EACvE,QAAQ,EACR,0BAA0B,GAAG,KAAK,EAClC,YAAY,GAAG,EAAE,EACjB,KAAK,EACL,QAAQ,EACR,oBAAoB,EACpB,uBAAuB,EACvB,mBAAmB,GACpB,EAAE,EAAE;IACH,MAAM,cAAc,GAAmB;QACrC,IAAI,EAAE,UAAU;QAChB,SAAS,EAAE,aAAa,CAAC,SAAS;QAClC,IAAI,EAAE,MAAM;QACZ,KAAK,EAAE,UAAU;QACjB,WAAW,EAAE,6BAA6B;QAC1C,QAAQ,EAAE,IAAI;QACd,IAAI,EAAE,CAAC;KACR,CAAC;IACF,OAAO,CACL,WAAK,KAAK,EAAC,eAAe,EAAC,GAAG,EAAE,QAAQ,CAAC,EAAE;QACzC,WAAK,KAAK,EAAC,6BAA6B;YACtC,oBACE,MAAM,EAAE,cAAc,EACtB,KAAK,EAAE,QAAQ,CAAC,QAAQ,EACxB,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CACnB,oBAAoB,CAAC;oBACnB,MAAM,EAAE;wBACN,UAAU,EAAE,QAAQ,CAAC,EAAE;wBACvB,GAAG,EAAE,UAAU;wBACf,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK;qBACtB;iBACiE,CAAC,GAEvE;YACF,oBACE,kBAAkB,EAAE,QAAQ,CAAC,WAAW,EAAE,OAAO,IAAI,KAAK,EAC1D,gBAAgB,EAAE,QAAQ,CAAC,WAAW,EAAE,KAAK,IAAI,EAAE,EACnD,mBAAmB,EAAE,CAAC,CAAQ,EAAE,EAAE;oBAChC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAI,CAA0C;yBACnE,MAAM,CAAC;oBACV,mBAAmB,CAAC;wBAClB,MAAM,EAAE;4BACN,UAAU,EAAE,QAAQ,CAAC,EAAE;4BACvB,OAAO;4BACP,KAAK;yBACN;qBACyC,CAAC,CAAC;gBAChD,CAAC,GACD;YACF,EAAC,uBAAuB,IACtB,UAAU,EAAE,QAAQ,CAAC,EAAE,EACvB,MAAM,EAAE,QAAQ,CAAC,eAAe,IAAI,EAAE,EACtC,0BAA0B,EAAE,0BAA0B,EACtD,YAAY,EAAE,YAAY,EAC1B,uBAAuB,EAAE,uBAAuB,GAChD,CACE;QAEN,EAAC,cAAc,IAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,GAAI;QAE1E,EAAC,iBAAiB,IAChB,MAAM,EAAE,QAAQ,CAAC,gBAAgB,EACjC,SAAS,EAAE,QAAQ,CAAC,SAAS,GAC7B;QAEF,EAAC,UAAU,IACT,SAAS,EAAE,QAAQ,CAAC,SAAS,EAC7B,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,EAClC,KAAK,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,EAC5B,QAAQ,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,GACrC,CACE,CACP,CAAC;AACJ,CAAC,CAAC","sourcesContent":["import { h, FunctionalComponent } from '@stencil/core';\nimport { TestCase } from '../../../types/llm-test-runner';\nimport { ResponseOutput } from './output/response-output';\nimport { EvaluationSummary } from './evaluation/evaluation-summary';\nimport { RowActions } from './actions/row-actions';\nimport { FormFieldType, TextAreaConfig } from '../../../lib/form/schema';\nimport {\n ExpectedOutcomeChangeDetail,\n ExpectedOutcomeRenderer,\n} from './expected-outcome-renderer';\nimport type { ChatHistoryChangeDetail } from './chat-history';\n\nexport type ChatHistoryRowChangeDetail = {\n testCaseId: string;\n} & ChatHistoryChangeDetail;\n\nexport interface LLMTestCaseRowProps {\n testCase: TestCase;\n dynamicResolutionSupported?: boolean;\n extractorIds?: string[];\n onRun: (testCase: TestCase) => void;\n onDelete: (id: string) => void;\n handleTestCaseChange: (\n e: CustomEvent<{ testCaseId: string; key: string; value: string }>,\n ) => void;\n onExpectedOutcomeChange: (\n e: CustomEvent<ExpectedOutcomeChangeDetail>,\n ) => void;\n onChatHistoryChange: (e: CustomEvent<ChatHistoryRowChangeDetail>) => void;\n}\n\nexport const LLMTestCaseRow: FunctionalComponent<LLMTestCaseRowProps> = ({\n testCase,\n dynamicResolutionSupported = false,\n extractorIds = [],\n onRun,\n onDelete,\n handleTestCaseChange,\n onExpectedOutcomeChange,\n onChatHistoryChange,\n}) => {\n const questionConfig: TextAreaConfig = {\n name: 'question',\n fieldType: FormFieldType.TEXT_AREA,\n type: 'text',\n label: 'Question',\n placeholder: 'Enter your question here...',\n required: true,\n rows: 3,\n };\n return (\n <div class=\"test-case-row\" key={testCase.id}>\n <div class=\"test-case-row__input-column\">\n <app-textarea\n config={questionConfig}\n value={testCase.question}\n onValueChange={(e) =>\n handleTestCaseChange({\n detail: {\n testCaseId: testCase.id,\n key: 'question',\n value: e.detail.value,\n },\n } as CustomEvent<{ testCaseId: string; key: string; value: string }>)\n }\n />\n <chat-history\n chatHistoryEnabled={testCase.chatHistory?.enabled ?? false}\n chatHistoryValue={testCase.chatHistory?.value ?? ''}\n onChatHistoryChange={(e: Event) => {\n const { enabled, value } = (e as CustomEvent<ChatHistoryChangeDetail>)\n .detail;\n onChatHistoryChange({\n detail: {\n testCaseId: testCase.id,\n enabled,\n value,\n },\n } as CustomEvent<ChatHistoryRowChangeDetail>);\n }}\n />\n <ExpectedOutcomeRenderer\n testCaseId={testCase.id}\n fields={testCase.expectedOutcome \|\| []}\n dynamicResolutionSupported={dynamicResolutionSupported}\n extractorIds={extractorIds}\n onExpectedOutcomeChange={onExpectedOutcomeChange}\n />\n </div>\n\n <ResponseOutput output={testCase.output} isRunning={testCase.isRunning} />\n\n <EvaluationSummary\n result={testCase.evaluationResult}\n isRunning={testCase.isRunning}\n />\n\n <RowActions\n isRunning={testCase.isRunning}\n canRun={!!testCase.question.trim()}\n onRun={() => onRun(testCase)}\n onDelete={() => onDelete(testCase.id)}\n />\n </div>\n );\n};\n"]}

package/dist/collection/components/llm-test-runner/test-cases/llm-test-cases.js CHANGED Viewed

@@ -1,7 +1,7 @@
 import { h } from "@stencil/core";
 import { LLMTestCaseRow } from "./llm-test-case-row";
 import { Button } from "../../../lib/ui/button/index";
-export const LLMTestCases = ({ testCases, dynamicResolutionSupported = false, onRun, onDelete, onAddTestCase, handleTestCaseChange, onExpectedOutcomeChange, onChatHistoryChange, }) => {
-    return (h("div", { class: "test-cases" }, h("div", { class: "test-cases__column-headers" }, h("div", { class: "test-cases__column-header" }, "Input"), h("div", { class: "test-cases__column-header" }, "Output"), h("div", { class: "test-cases__column-header" }, "Evaluation"), h("div", { class: "test-cases__column-header" }, "Actions")), testCases.map(testCase => (h(LLMTestCaseRow, { testCase: testCase, dynamicResolutionSupported: dynamicResolutionSupported, onRun: onRun, onDelete: onDelete, handleTestCaseChange: handleTestCaseChange, onExpectedOutcomeChange: onExpectedOutcomeChange, onChatHistoryChange: onChatHistoryChange }))), h("div", { class: "test-cases__add-section" }, h(Button, { variant: "outline", size: "md", onClick: onAddTestCase }, "+ Add Question"))));
+export const LLMTestCases = ({ testCases, dynamicResolutionSupported = false, extractorIds = [], onRun, onDelete, onAddTestCase, handleTestCaseChange, onExpectedOutcomeChange, onChatHistoryChange, }) => {
+    return (h("div", { class: "test-cases" }, h("div", { class: "test-cases__column-headers" }, h("div", { class: "test-cases__column-header" }, "Input"), h("div", { class: "test-cases__column-header" }, "Output"), h("div", { class: "test-cases__column-header" }, "Evaluation"), h("div", { class: "test-cases__column-header" }, "Actions")), testCases.map(testCase => (h(LLMTestCaseRow, { testCase: testCase, dynamicResolutionSupported: dynamicResolutionSupported, extractorIds: extractorIds, onRun: onRun, onDelete: onDelete, handleTestCaseChange: handleTestCaseChange, onExpectedOutcomeChange: onExpectedOutcomeChange, onChatHistoryChange: onChatHistoryChange }))), h("div", { class: "test-cases__add-section" }, h(Button, { variant: "outline", size: "md", onClick: onAddTestCase }, "+ Add Question"))));
 };
 //# sourceMappingURL=llm-test-cases.js.map

package/dist/collection/components/llm-test-runner/test-cases/llm-test-cases.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"llm-test-cases.js","sourceRoot":"","sources":["../../../../src/components/llm-test-runner/test-cases/llm-test-cases.tsx"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAuB,MAAM,eAAe,CAAC;AAEvD,OAAO,EAAE,cAAc,EAA8B,MAAM,qBAAqB,CAAC;AACjF,OAAO,EAAE,MAAM,EAAE,MAAM,8BAA8B,CAAC;~~AAkBtD~~,MAAM,CAAC,MAAM,YAAY,GAA2C,CAAC,EACnE,SAAS,EACT,0BAA0B,GAAG,KAAK,EAClC,KAAK,EACL,QAAQ,EACR,aAAa,EACb,oBAAoB,EACpB,uBAAuB,EACvB,mBAAmB,GACpB,EAAE,EAAE;IACH,OAAO,CACL,WAAK,KAAK,EAAC,YAAY;QACrB,WAAK,KAAK,EAAC,4BAA4B;YACrC,WAAK,KAAK,EAAC,2BAA2B,YAAY;YAClD,WAAK,KAAK,EAAC,2BAA2B,aAAa;YACnD,WAAK,KAAK,EAAC,2BAA2B,iBAAiB;YACvD,WAAK,KAAK,EAAC,2BAA2B,cAAc,CAChD;QAEL,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CACzB,EAAC,cAAc,IACb,QAAQ,EAAE,QAAQ,EAClB,0BAA0B,EAAE,0BAA0B,EACtD,KAAK,EAAE,KAAK,EACZ,QAAQ,EAAE,QAAQ,EAClB,oBAAoB,EAAE,oBAAoB,EAC1C,uBAAuB,EAAE,uBAAuB,EAChD,mBAAmB,EAAE,mBAAmB,GACxC,CACH,CAAC;QAEF,WAAK,KAAK,EAAC,yBAAyB;YAClC,EAAC,MAAM,IAAC,OAAO,EAAC,SAAS,EAAC,IAAI,EAAC,IAAI,EAAC,OAAO,EAAE,aAAa,qBAEjD,CACL,CACF,CACP,CAAC;AACJ,CAAC,CAAC","sourcesContent":["import { h, FunctionalComponent } from '@stencil/core';\nimport { TestCase } from '../../../types/llm-test-runner';\nimport { LLMTestCaseRow, ChatHistoryRowChangeDetail } from './llm-test-case-row';\nimport { Button } from '../../../lib/ui/button/index';\nimport { ExpectedOutcomeChangeDetail } from './expected-outcome-renderer';\n\nexport interface LLMTestCasesProps {\n testCases: TestCase[];\n dynamicResolutionSupported?: boolean;\n onRun: (testCase: TestCase) => void;\n onDelete: (id: string) => void;\n onAddTestCase: () => void;\n handleTestCaseChange: (\n e: CustomEvent<{ testCaseId: string; key: string; value: string }>,\n ) => void;\n onExpectedOutcomeChange: (\n e: CustomEvent<ExpectedOutcomeChangeDetail>,\n ) => void;\n onChatHistoryChange: (e: CustomEvent<ChatHistoryRowChangeDetail>) => void;\n}\n\nexport const LLMTestCases: FunctionalComponent<LLMTestCasesProps> = ({\n testCases,\n dynamicResolutionSupported = false,\n onRun,\n onDelete,\n onAddTestCase,\n handleTestCaseChange,\n onExpectedOutcomeChange,\n onChatHistoryChange,\n}) => {\n return (\n <div class=\"test-cases\">\n <div class=\"test-cases__column-headers\">\n <div class=\"test-cases__column-header\">Input</div>\n <div class=\"test-cases__column-header\">Output</div>\n <div class=\"test-cases__column-header\">Evaluation</div>\n <div class=\"test-cases__column-header\">Actions</div>\n </div>\n\n {testCases.map(testCase => (\n <LLMTestCaseRow\n testCase={testCase}\n dynamicResolutionSupported={dynamicResolutionSupported}\n onRun={onRun}\n onDelete={onDelete}\n handleTestCaseChange={handleTestCaseChange}\n onExpectedOutcomeChange={onExpectedOutcomeChange}\n onChatHistoryChange={onChatHistoryChange}\n />\n ))}\n\n <div class=\"test-cases__add-section\">\n <Button variant=\"outline\" size=\"md\" onClick={onAddTestCase}>\n + Add Question\n </Button>\n </div>\n </div>\n );\n};\n"]}
1	+ {"version":3,"file":"llm-test-cases.js","sourceRoot":"","sources":["../../../../src/components/llm-test-runner/test-cases/llm-test-cases.tsx"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAuB,MAAM,eAAe,CAAC;AAEvD,OAAO,EAAE,cAAc,EAA8B,MAAM,qBAAqB,CAAC;AACjF,OAAO,EAAE,MAAM,EAAE,MAAM,8BAA8B,CAAC;AAmBtD,MAAM,CAAC,MAAM,YAAY,GAA2C,CAAC,EACnE,SAAS,EACT,0BAA0B,GAAG,KAAK,EAClC,YAAY,GAAG,EAAE,EACjB,KAAK,EACL,QAAQ,EACR,aAAa,EACb,oBAAoB,EACpB,uBAAuB,EACvB,mBAAmB,GACpB,EAAE,EAAE;IACH,OAAO,CACL,WAAK,KAAK,EAAC,YAAY;QACrB,WAAK,KAAK,EAAC,4BAA4B;YACrC,WAAK,KAAK,EAAC,2BAA2B,YAAY;YAClD,WAAK,KAAK,EAAC,2BAA2B,aAAa;YACnD,WAAK,KAAK,EAAC,2BAA2B,iBAAiB;YACvD,WAAK,KAAK,EAAC,2BAA2B,cAAc,CAChD;QAEL,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CACzB,EAAC,cAAc,IACb,QAAQ,EAAE,QAAQ,EAClB,0BAA0B,EAAE,0BAA0B,EACtD,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,KAAK,EACZ,QAAQ,EAAE,QAAQ,EAClB,oBAAoB,EAAE,oBAAoB,EAC1C,uBAAuB,EAAE,uBAAuB,EAChD,mBAAmB,EAAE,mBAAmB,GACxC,CACH,CAAC;QAEF,WAAK,KAAK,EAAC,yBAAyB;YAClC,EAAC,MAAM,IAAC,OAAO,EAAC,SAAS,EAAC,IAAI,EAAC,IAAI,EAAC,OAAO,EAAE,aAAa,qBAEjD,CACL,CACF,CACP,CAAC;AACJ,CAAC,CAAC","sourcesContent":["import { h, FunctionalComponent } from '@stencil/core';\nimport { TestCase } from '../../../types/llm-test-runner';\nimport { LLMTestCaseRow, ChatHistoryRowChangeDetail } from './llm-test-case-row';\nimport { Button } from '../../../lib/ui/button/index';\nimport { ExpectedOutcomeChangeDetail } from './expected-outcome-renderer';\n\nexport interface LLMTestCasesProps {\n testCases: TestCase[];\n dynamicResolutionSupported?: boolean;\n extractorIds?: string[];\n onRun: (testCase: TestCase) => void;\n onDelete: (id: string) => void;\n onAddTestCase: () => void;\n handleTestCaseChange: (\n e: CustomEvent<{ testCaseId: string; key: string; value: string }>,\n ) => void;\n onExpectedOutcomeChange: (\n e: CustomEvent<ExpectedOutcomeChangeDetail>,\n ) => void;\n onChatHistoryChange: (e: CustomEvent<ChatHistoryRowChangeDetail>) => void;\n}\n\nexport const LLMTestCases: FunctionalComponent<LLMTestCasesProps> = ({\n testCases,\n dynamicResolutionSupported = false,\n extractorIds = [],\n onRun,\n onDelete,\n onAddTestCase,\n handleTestCaseChange,\n onExpectedOutcomeChange,\n onChatHistoryChange,\n}) => {\n return (\n <div class=\"test-cases\">\n <div class=\"test-cases__column-headers\">\n <div class=\"test-cases__column-header\">Input</div>\n <div class=\"test-cases__column-header\">Output</div>\n <div class=\"test-cases__column-header\">Evaluation</div>\n <div class=\"test-cases__column-header\">Actions</div>\n </div>\n\n {testCases.map(testCase => (\n <LLMTestCaseRow\n testCase={testCase}\n dynamicResolutionSupported={dynamicResolutionSupported}\n extractorIds={extractorIds}\n onRun={onRun}\n onDelete={onDelete}\n handleTestCaseChange={handleTestCaseChange}\n onExpectedOutcomeChange={onExpectedOutcomeChange}\n onChatHistoryChange={onChatHistoryChange}\n />\n ))}\n\n <div class=\"test-cases__add-section\">\n <Button variant=\"outline\" size=\"md\" onClick={onAddTestCase}>\n + Add Question\n </Button>\n </div>\n </div>\n );\n};\n"]}

package/dist/collection/components/llm-test-runner/test-cases/output/response-output.js CHANGED Viewed

@@ -1,5 +1,5 @@
 import { h } from "@stencil/core";
 export const ResponseOutput = ({ output, isRunning, }) => {
-    return (h("div", { class: "response-output" }, output ? (h("div", { class: "response-output__content" }, output)) : (h("div", { class: "response-output__placeholder" }, isRunning ? 'Running...' : ''))));
+    return (h("div", { class: "response-output" }, output?.text ? (h("div", { class: "response-output__content" }, output.text)) : (h("div", { class: "response-output__placeholder" }, isRunning ? 'Running...' : ''))));
 };
 //# sourceMappingURL=response-output.js.map

package/dist/collection/components/llm-test-runner/test-cases/output/response-output.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"response-output.js","sourceRoot":"","sources":["../../../../../src/components/llm-test-runner/test-cases/output/response-output.tsx"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAuB,MAAM,eAAe,CAAC;~~AAOvD~~,MAAM,CAAC,MAAM,cAAc,GAA6C,CAAC,EACvE,MAAM,EACN,SAAS,GACV,EAAE,EAAE;IACH,OAAO,CACL,WAAK,KAAK,EAAC,iBAAiB,IACzB,MAAM,CAAC,CAAC,CAAC,~~CACR~~,WAAK,KAAK,EAAC,0BAA0B,IAAE,MAAM,CAAO,~~CACrD~~,CAAC,CAAC,CAAC,CACF,WAAK,KAAK,EAAC,8BAA8B,IAAE,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAO,CAChF,CACG,CACP,CAAC;AACJ,CAAC,CAAC","sourcesContent":["import { h, FunctionalComponent } from '@stencil/core';\n\nexport interface ResponseOutputProps {\n output?: ~~string~~;\n isRunning: boolean;\n}\n\nexport const ResponseOutput: FunctionalComponent<ResponseOutputProps> = ({\n output,\n isRunning,\n}) => {\n return (\n <div class=\"response-output\">\n {output ? (\n <div class=\"response-output__content\">{output}</div>\n ) : (\n <div class=\"response-output__placeholder\">{isRunning ? 'Running...' : ''}</div>\n )}\n </div>\n );\n};\n\n"]}
1	+ {"version":3,"file":"response-output.js","sourceRoot":"","sources":["../../../../../src/components/llm-test-runner/test-cases/output/response-output.tsx"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAuB,MAAM,eAAe,CAAC;AAQvD,MAAM,CAAC,MAAM,cAAc,GAA6C,CAAC,EACvE,MAAM,EACN,SAAS,GACV,EAAE,EAAE;IACH,OAAO,CACL,WAAK,KAAK,EAAC,iBAAiB,IACzB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CACd,WAAK,KAAK,EAAC,0BAA0B,IAAE,MAAM,CAAC,IAAI,CAAO,CAC1D,CAAC,CAAC,CAAC,CACF,WAAK,KAAK,EAAC,8BAA8B,IAAE,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAO,CAChF,CACG,CACP,CAAC;AACJ,CAAC,CAAC","sourcesContent":["import { h, FunctionalComponent } from '@stencil/core';\nimport type { ModelResponsePayload } from '../../../../types/llm-test-runner';\n\nexport interface ResponseOutputProps {\n output?: ModelResponsePayload;\n isRunning: boolean;\n}\n\nexport const ResponseOutput: FunctionalComponent<ResponseOutputProps> = ({\n output,\n isRunning,\n}) => {\n return (\n <div class=\"response-output\">\n {output?.text ? (\n <div class=\"response-output__content\">{output.text}</div>\n ) : (\n <div class=\"response-output__placeholder\">{isRunning ? 'Running...' : ''}</div>\n )}\n </div>\n );\n};\n\n"]}

package/dist/collection/demo/demo-modes.js ADDED Viewed

@@ -0,0 +1,130 @@
+export const DEMO_MODES = {
+  simpleTest: {
+    initialTestCases: [
+      {
+        id: 'demo-simpleTest-1',
+        question:
+          'What is the capital of France? Reply with just the city name, nothing else.',
+        expectedOutcome: [
+          {
+            type: 'textarea',
+            label: 'Expected Outcome',
+            value: 'paris',
+            evaluationParameters: { approach: 'exact' },
+          },
+        ],
+      },
+      {
+        id: 'demo-simpleTest-2',
+        question: 'How many legs does a typical dog have? One short phrase is fine.',
+        expectedOutcome: [
+          {
+            type: 'textarea',
+            label: 'Expected Outcome',
+            value: '4,four',
+            evaluationParameters: {
+              approach: 'semantic',
+              threshold: 0.48,
+            },
+          },
+        ],
+      },
+    ],
+  },
+  multipleExpectedOutcomes: {
+    defaultExpectedOutcomeSchema: [
+      {
+        type: 'text',
+        label: 'Must include',
+        placeholder: 'Word that should appear in the answer',
+        evaluationParameters: { approach: 'exact' },
+      },
+      {
+        type: 'textarea',
+        label: 'Meaning check',
+        placeholder: 'Comma-separated ideas to match loosely',
+        rows: 3,
+        evaluationParameters: { approach: 'semantic', threshold: 0.5 },
+      },
+      {
+        type: 'chips-input',
+        label: 'Keywords',
+        placeholder: 'Add chip',
+        evaluationParameters: { approach: 'exact' },
+      },
+      {
+        type: 'select',
+        label: 'Says yes or no',
+        placeholder: 'Pick one',
+        options: ['Yes', 'No'],
+        evaluationParameters: { approach: 'exact' },
+      },
+    ],
+    initialTestCases: [
+      {
+        id: 'demo-multipleExpectedOutcomes-1',
+        question:
+          'Is water a liquid? Answer yes or no in one short sentence, and use the word water.',
+        expectedOutcome: [
+          {
+            type: 'text',
+            label: 'Must include',
+            value: 'water',
+            evaluationParameters: { approach: 'exact' },
+          },
+          {
+            type: 'textarea',
+            label: 'Meaning check',
+            value: 'yes,liquid',
+            evaluationParameters: {
+              approach: 'semantic',
+              threshold: 0.5,
+            },
+          },
+          {
+            type: 'chips-input',
+            label: 'Keywords',
+            value: ['yes', 'water'],
+            evaluationParameters: { approach: 'exact' },
+          },
+          {
+            type: 'select',
+            label: 'Says yes or no',
+            options: ['Yes', 'No'],
+            value: 'Yes',
+            evaluationParameters: { approach: 'exact' },
+          },
+        ],
+      },
+    ],
+  },
+  dynamicExpectedOutcome: {
+    initialTestCases: [
+      {
+        id: 'demo-dynamicExpectedOutcome-1',
+        question:
+          'In one sentence, when can customers reach support?',
+        expectedOutcome: [
+          {
+            type: 'textarea',
+            label: 'Gold answer (filled when you run)',
+            outcomeMode: 'dynamic',
+            resolutionQuery: 'demo.faq.hours',
+            value: '',
+            evaluationParameters: {
+              approach: 'semantic',
+              threshold: 0.62,
+            },
+          },
+        ],
+      },
+    ],
+    resolveExpectedOutcome: async (resolutionQuery) => {
+      const key = resolutionQuery.trim();
+      if (key === 'demo.faq.hours') {
+        return 'Support is available 9am–5pm Eastern Time, Monday through Friday.';
+      }
+      return `Unknown resolution key: ${key}`;
+    },
+  },
+};

package/dist/collection/demo/vanilla-demo.js ADDED Viewed

@@ -0,0 +1,56 @@
+import { DEMO_MODES } from './demo-modes.js';
+function wireGemini(runner) {
+  const llm = new window.GeminiAdapter(window.env.API_KEY);
+  runner.addEventListener('llmRequest', async (event) => {
+    try {
+      const response = await llm.invoke(event.detail.prompt);
+      event.detail.resolve({ text: response });
+    } catch (err) {
+      event.detail.reject(
+        err instanceof Error ? err : new Error(String(err)),
+      );
+    }
+  });
+}
+function mountMode(host, modeKey) {
+  const config = DEMO_MODES[modeKey];
+  host.replaceChildren();
+  const runner = document.createElement('llm-test-runner');
+  runner.delayMs = 1000;
+  runner.initialTestCases = config.initialTestCases;
+  if (config.defaultExpectedOutcomeSchema) {
+    runner.defaultExpectedOutcomeSchema = config.defaultExpectedOutcomeSchema;
+  }
+  if (config.resolveExpectedOutcome) {
+    runner.resolveExpectedOutcome = config.resolveExpectedOutcome;
+  }
+  host.appendChild(runner);
+  wireGemini(runner);
+}
+function initVanillaDemo() {
+  const host = document.getElementById('runner-host');
+  const modeSelect = document.getElementById('demo-example-mode');
+  function showMode(modeKey) {
+    mountMode(host, modeKey);
+  }
+  modeSelect.addEventListener('change', () => {
+    showMode(modeSelect.value);
+  });
+  void (async () => {
+    await customElements.whenDefined('llm-test-runner');
+    modeSelect.value = 'simpleTest';
+    showMode('simpleTest');
+  })();
+}
+if (document.readyState === 'loading') {
+  document.addEventListener('DOMContentLoaded', initVanillaDemo);
+} else {
+  initVanillaDemo();
+}

package/dist/collection/lib/evaluation/actual-value-resolver.js ADDED Viewed

@@ -0,0 +1,52 @@
+function toTextSource() {
+    return { type: 'text' };
+}
+export async function resolveActualValue(field, output, extractors) {
+    const source = field.evaluationSource || toTextSource();
+    if (source.type === 'text') {
+        const text = output?.text?.trim();
+        if (!text) {
+            return {
+                success: false,
+                error: 'Model response text is empty.',
+            };
+        }
+        return { success: true, value: text };
+    }
+    const extractor = extractors?.[source.extractorId];
+    if (!extractor) {
+        return {
+            success: false,
+            error: `Extractor "${source.extractorId}" is not registered.`,
+        };
+    }
+    try {
+        const extractedRaw = await extractor(output || {});
+        if (typeof extractedRaw !== 'string') {
+            return {
+                success: false,
+                error: `Extractor "${source.extractorId}" must return a string.`,
+            };
+        }
+        const extracted = extractedRaw.trim();
+        if (!extracted) {
+            return {
+                success: false,
+                error: `Extractor "${source.extractorId}" returned an empty value.`,
+            };
+        }
+        return {
+            success: true,
+            value: extracted,
+        };
+    }
+    catch (error) {
+        return {
+            success: false,
+            error: error instanceof Error
+                ? error.message
+                : `Extractor "${source.extractorId}" failed.`,
+        };
+    }
+}
+//# sourceMappingURL=actual-value-resolver.js.map

package/dist/collection/lib/evaluation/actual-value-resolver.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"actual-value-resolver.js","sourceRoot":"","sources":["../../../src/lib/evaluation/actual-value-resolver.ts"],"names":[],"mappings":"AAUA,SAAS,YAAY;IACnB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAW,CAAC;AACnC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,KAA2B,EAC3B,MAA6B,EAC7B,UAAuC;IAEvC,MAAM,MAAM,GAAG,KAAK,CAAC,gBAAgB,IAAI,YAAY,EAAE,CAAC;IAExD,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,+BAA+B;aACvC,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IACxC,CAAC;IAED,MAAM,SAAS,GAAG,UAAU,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IACnD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,cAAc,MAAM,CAAC,WAAW,sBAAsB;SAC9D,CAAC;IACJ,CAAC;IAED,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QACnD,IAAI,OAAO,YAAY,KAAK,QAAQ,EAAE,CAAC;YACrC,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,cAAc,MAAM,CAAC,WAAW,yBAAyB;aACjE,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,YAAY,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,cAAc,MAAM,CAAC,WAAW,4BAA4B;aACpE,CAAC;QACJ,CAAC;QAED,OAAO;YACL,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,SAAS;SACjB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EACH,KAAK,YAAY,KAAK;gBACpB,CAAC,CAAC,KAAK,CAAC,OAAO;gBACf,CAAC,CAAC,cAAc,MAAM,CAAC,WAAW,WAAW;SAClD,CAAC;IACJ,CAAC;AACH,CAAC","sourcesContent":["import type {\n EvaluationSourceExtractors,\n ExpectedOutcomeField,\n ModelResponsePayload,\n} from '../../types/llm-test-runner';\n\nexport type ResolvedActualValue =\n | { success: true; value: string }\n | { success: false; error: string };\n\nfunction toTextSource() {\n return { type: 'text' } as const;\n}\n\nexport async function resolveActualValue(\n field: ExpectedOutcomeField,\n output?: ModelResponsePayload,\n extractors?: EvaluationSourceExtractors,\n): Promise<ResolvedActualValue> {\n const source = field.evaluationSource || toTextSource();\n\n if (source.type === 'text') {\n const text = output?.text?.trim();\n if (!text) {\n return {\n success: false,\n error: 'Model response text is empty.',\n };\n }\n return { success: true, value: text };\n }\n\n const extractor = extractors?.[source.extractorId];\n if (!extractor) {\n return {\n success: false,\n error: `Extractor \"${source.extractorId}\" is not registered.`,\n };\n }\n\n try {\n const extractedRaw = await extractor(output || {});\n if (typeof extractedRaw !== 'string') {\n return {\n success: false,\n error: `Extractor \"${source.extractorId}\" must return a string.`,\n };\n }\n\n const extracted = extractedRaw.trim();\n if (!extracted) {\n return {\n success: false,\n error: `Extractor \"${source.extractorId}\" returned an empty value.`,\n };\n }\n\n return {\n success: true,\n value: extracted,\n };\n } catch (error) {\n return {\n success: false,\n error:\n error instanceof Error\n ? error.message\n : `Extractor \"${source.extractorId}\" failed.`,\n };\n }\n}\n"]}

package/dist/collection/lib/evaluation/evaluation-engine.js CHANGED Viewed

@@ -10,7 +10,7 @@ export class LLMEvaluationEngine {
             const fieldRequest = {
                 testCaseId: request.testCaseId,
                 question: request.question,
-                actualResponse: request.actualResponse,
+                actualResponse: field.actualResponse,
                 expectedOutcome: field.expectedValue,
                 evaluationParameters: field.evaluationParameters,
             };

package/dist/collection/lib/evaluation/evaluation-engine.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"evaluation-engine.js","sourceRoot":"","sources":["../../../src/lib/evaluation/evaluation-engine.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AACxE,OAAO,EAAE,yBAAyB,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AACxE,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AAEzE,MAAM,OAAO,mBAAmB;IAC9B,KAAK,CAAC,gBAAgB,CACpB,OAA4B,EAC5B,QAA4B;QAE5B,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,UAAU,CAC7C,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAC,KAAK,EAAC,EAAE;YAC/B,MAAM,YAAY,GAAsB;gBACtC,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,cAAc,EAAE,~~OAAO~~,CAAC,cAAc;~~gBACtC~~,eAAe,EAAE,KAAK,CAAC,aAAa;gBACpC,oBAAoB,EAAE,KAAK,CAAC,oBAAoB;aACjD,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC;YAEtD,MAAM,WAAW,GAA0B;gBACzC,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,aAAa,EAAE,KAAK,CAAC,aAAa;gBAClC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,oBAAoB,EAAE,MAAM,CAAC,oBAAqB;gBAClD,wBAAwB,EAAE,MAAM,CAAC,wBAAwB;aAC1D,CAAC;YACF,OAAO,WAAW,CAAC;QACrB,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,YAAY,GAA4B,cAAc,CAAC,GAAG,CAC9D,CAAC,aAAa,EAAE,KAAK,EAAE,EAAE;YACvB,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACpC,IAAI,aAAa,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBACzC,OAAO,aAAa,CAAC,KAAK,CAAC;YAC7B,CAAC;YAED,OAAO;gBACL,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,aAAa,EAAE,KAAK,CAAC,aAAa;gBAClC,MAAM,EAAE,KAAK;gBACb,cAAc,EAAE,EAAE;gBAClB,oBAAoB,EAAE,KAAK,CAAC,oBAAoB;gBAChD,wBAAwB,EAAE;oBACxB,KAAK,EAAE,CAAC;oBACR,YAAY,EAAE,KAAK,CAAC,oBAAoB,CAAC,QAAQ;iBAClD;gBACD,KAAK,EAAE,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,MAAM,CAAC;aACtD,CAAC;QACJ,CAAC,CACF,CAAC;QAEF,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAC3E,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEzE,QAAQ,CAAC;YACP,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,MAAM;YACN,cAAc;YACd,YAAY;YACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAA0B;QACpD,MAAM,QAAQ,GAAuB,OAAO,CAAC,oBAAoB,CAAC,QAAQ,CAAC;QAC3E,QAAQ,QAAQ,EAAE,CAAC;YACjB,KAAK,kBAAkB,CAAC,IAAI;gBAC1B,OAAO,qBAAqB,CAAC,OAAO,CAAC,CAAC;YACxC,KAAK,kBAAkB,CAAC,KAAK;gBAC3B,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;YACpC,KAAK,kBAAkB,CAAC,OAAO;gBAC7B,OAAO,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAC1C,KAAK,kBAAkB,CAAC,OAAO;gBAC7B,OAAO,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAC1C,KAAK,kBAAkB,CAAC,QAAQ;gBAC9B,OAAO,yBAAyB,CAAC,OAAO,CAAC,CAAC;YAC5C;gBACE,OAAO,CAAC,IAAI,CACV,8BAA8B,OAAO,CAAC,oBAAoB,CAAC,QAAQ,kCAAkC,CACtG,CAAC;gBACF,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAEO,mBAAmB,CAAC,KAAc;QACxC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,0BAA0B,CAAC;IAC7E,CAAC;CACF","sourcesContent":["import {\n EvaluationRequest,\n EvaluationResult,\n EvaluationCallback,\n FieldEvaluationResult,\n EvaluationRequestV2,\n} from './types';\nimport { performEvaluation } from './evaluators/exact/exact';\nimport { EvaluationApproach } from './constants';\nimport { performRouge1Evaluation } from './evaluators/rouge1-evaluator';\nimport { performSemanticEvaluation } from './evaluators/semantic/index';\nimport { performRougeLEvaluation } from './evaluators/rougeL-evaluator';\nimport { performBleuEvaluation } from './evaluators/bleu/bleu-evaluator';\n\nexport class LLMEvaluationEngine {\n async evaluateResponse(\n request: EvaluationRequestV2,\n callback: EvaluationCallback,\n ): Promise<void> {\n const settledResults = await Promise.allSettled(\n request.fields.map(async field => {\n const fieldRequest: EvaluationRequest = {\n testCaseId: request.testCaseId,\n question: request.question,\n actualResponse: ~~request~~.actualResponse,\n expectedOutcome: field.expectedValue,\n evaluationParameters: field.evaluationParameters,\n };\n const result = await this.evaluateField(fieldRequest);\n\n const fieldResult: FieldEvaluationResult = {\n index: field.index,\n label: field.label,\n type: field.type,\n expectedValue: field.expectedValue,\n passed: result.passed,\n keywordMatches: result.keywordMatches,\n evaluationParameters: result.evaluationParameters!,\n evaluationApproachResult: result.evaluationApproachResult,\n };\n return fieldResult;\n }),\n );\n\n const fieldResults: FieldEvaluationResult[] = settledResults.map(\n (settledResult, index) => {\n const field = request.fields[index];\n if (settledResult.status === 'fulfilled') {\n return settledResult.value;\n }\n\n return {\n index: field.index,\n label: field.label,\n type: field.type,\n expectedValue: field.expectedValue,\n passed: false,\n keywordMatches: [],\n evaluationParameters: field.evaluationParameters,\n evaluationApproachResult: {\n score: 0,\n approachUsed: field.evaluationParameters.approach,\n },\n error: this.getSafeErrorMessage(settledResult.reason),\n };\n },\n );\n\n const keywordMatches = fieldResults.flatMap(field => field.keywordMatches);\n const passed = fieldResults.every(field => field.passed && !field.error);\n\n callback({\n testCaseId: request.testCaseId,\n passed,\n keywordMatches,\n fieldResults,\n timestamp: new Date().toISOString(),\n });\n }\n\n private async evaluateField(request: EvaluationRequest): Promise<EvaluationResult> {\n const approach: EvaluationApproach = request.evaluationParameters.approach;\n switch (approach) {\n case EvaluationApproach.BLEU:\n return performBleuEvaluation(request);\n case EvaluationApproach.EXACT:\n return performEvaluation(request);\n case EvaluationApproach.ROUGE_1:\n return performRouge1Evaluation(request);\n case EvaluationApproach.ROUGE_L:\n return performRougeLEvaluation(request);\n case EvaluationApproach.SEMANTIC:\n return performSemanticEvaluation(request);\n default:\n console.warn(\n `Unknown matching approach: ${request.evaluationParameters.approach}, falling back to exact matching`,\n );\n return performEvaluation(request);\n }\n }\n\n private getSafeErrorMessage(error: unknown): string {\n return error instanceof Error ? error.message : 'Field evaluation failed.';\n }\n}\n"]}
1	+ {"version":3,"file":"evaluation-engine.js","sourceRoot":"","sources":["../../../src/lib/evaluation/evaluation-engine.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AACxE,OAAO,EAAE,yBAAyB,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AACxE,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AAEzE,MAAM,OAAO,mBAAmB;IAC9B,KAAK,CAAC,gBAAgB,CACpB,OAA4B,EAC5B,QAA4B;QAE5B,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,UAAU,CAC7C,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAC,KAAK,EAAC,EAAE;YAC/B,MAAM,YAAY,GAAsB;gBACtC,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,cAAc,EAAE,KAAK,CAAC,cAAc;gBACpC,eAAe,EAAE,KAAK,CAAC,aAAa;gBACpC,oBAAoB,EAAE,KAAK,CAAC,oBAAoB;aACjD,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC;YAEtD,MAAM,WAAW,GAA0B;gBACzC,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,aAAa,EAAE,KAAK,CAAC,aAAa;gBAClC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,oBAAoB,EAAE,MAAM,CAAC,oBAAqB;gBAClD,wBAAwB,EAAE,MAAM,CAAC,wBAAwB;aAC1D,CAAC;YACF,OAAO,WAAW,CAAC;QACrB,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,YAAY,GAA4B,cAAc,CAAC,GAAG,CAC9D,CAAC,aAAa,EAAE,KAAK,EAAE,EAAE;YACvB,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACpC,IAAI,aAAa,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBACzC,OAAO,aAAa,CAAC,KAAK,CAAC;YAC7B,CAAC;YAED,OAAO;gBACL,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,aAAa,EAAE,KAAK,CAAC,aAAa;gBAClC,MAAM,EAAE,KAAK;gBACb,cAAc,EAAE,EAAE;gBAClB,oBAAoB,EAAE,KAAK,CAAC,oBAAoB;gBAChD,wBAAwB,EAAE;oBACxB,KAAK,EAAE,CAAC;oBACR,YAAY,EAAE,KAAK,CAAC,oBAAoB,CAAC,QAAQ;iBAClD;gBACD,KAAK,EAAE,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,MAAM,CAAC;aACtD,CAAC;QACJ,CAAC,CACF,CAAC;QAEF,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAC3E,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEzE,QAAQ,CAAC;YACP,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,MAAM;YACN,cAAc;YACd,YAAY;YACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAA0B;QACpD,MAAM,QAAQ,GAAuB,OAAO,CAAC,oBAAoB,CAAC,QAAQ,CAAC;QAC3E,QAAQ,QAAQ,EAAE,CAAC;YACjB,KAAK,kBAAkB,CAAC,IAAI;gBAC1B,OAAO,qBAAqB,CAAC,OAAO,CAAC,CAAC;YACxC,KAAK,kBAAkB,CAAC,KAAK;gBAC3B,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;YACpC,KAAK,kBAAkB,CAAC,OAAO;gBAC7B,OAAO,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAC1C,KAAK,kBAAkB,CAAC,OAAO;gBAC7B,OAAO,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAC1C,KAAK,kBAAkB,CAAC,QAAQ;gBAC9B,OAAO,yBAAyB,CAAC,OAAO,CAAC,CAAC;YAC5C;gBACE,OAAO,CAAC,IAAI,CACV,8BAA8B,OAAO,CAAC,oBAAoB,CAAC,QAAQ,kCAAkC,CACtG,CAAC;gBACF,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAEO,mBAAmB,CAAC,KAAc;QACxC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,0BAA0B,CAAC;IAC7E,CAAC;CACF","sourcesContent":["import {\n EvaluationRequest,\n EvaluationResult,\n EvaluationCallback,\n FieldEvaluationResult,\n EvaluationRequestV2,\n} from './types';\nimport { performEvaluation } from './evaluators/exact/exact';\nimport { EvaluationApproach } from './constants';\nimport { performRouge1Evaluation } from './evaluators/rouge1-evaluator';\nimport { performSemanticEvaluation } from './evaluators/semantic/index';\nimport { performRougeLEvaluation } from './evaluators/rougeL-evaluator';\nimport { performBleuEvaluation } from './evaluators/bleu/bleu-evaluator';\n\nexport class LLMEvaluationEngine {\n async evaluateResponse(\n request: EvaluationRequestV2,\n callback: EvaluationCallback,\n ): Promise<void> {\n const settledResults = await Promise.allSettled(\n request.fields.map(async field => {\n const fieldRequest: EvaluationRequest = {\n testCaseId: request.testCaseId,\n question: request.question,\n actualResponse: field.actualResponse,\n expectedOutcome: field.expectedValue,\n evaluationParameters: field.evaluationParameters,\n };\n const result = await this.evaluateField(fieldRequest);\n\n const fieldResult: FieldEvaluationResult = {\n index: field.index,\n label: field.label,\n type: field.type,\n expectedValue: field.expectedValue,\n passed: result.passed,\n keywordMatches: result.keywordMatches,\n evaluationParameters: result.evaluationParameters!,\n evaluationApproachResult: result.evaluationApproachResult,\n };\n return fieldResult;\n }),\n );\n\n const fieldResults: FieldEvaluationResult[] = settledResults.map(\n (settledResult, index) => {\n const field = request.fields[index];\n if (settledResult.status === 'fulfilled') {\n return settledResult.value;\n }\n\n return {\n index: field.index,\n label: field.label,\n type: field.type,\n expectedValue: field.expectedValue,\n passed: false,\n keywordMatches: [],\n evaluationParameters: field.evaluationParameters,\n evaluationApproachResult: {\n score: 0,\n approachUsed: field.evaluationParameters.approach,\n },\n error: this.getSafeErrorMessage(settledResult.reason),\n };\n },\n );\n\n const keywordMatches = fieldResults.flatMap(field => field.keywordMatches);\n const passed = fieldResults.every(field => field.passed && !field.error);\n\n callback({\n testCaseId: request.testCaseId,\n passed,\n keywordMatches,\n fieldResults,\n timestamp: new Date().toISOString(),\n });\n }\n\n private async evaluateField(request: EvaluationRequest): Promise<EvaluationResult> {\n const approach: EvaluationApproach = request.evaluationParameters.approach;\n switch (approach) {\n case EvaluationApproach.BLEU:\n return performBleuEvaluation(request);\n case EvaluationApproach.EXACT:\n return performEvaluation(request);\n case EvaluationApproach.ROUGE_1:\n return performRouge1Evaluation(request);\n case EvaluationApproach.ROUGE_L:\n return performRougeLEvaluation(request);\n case EvaluationApproach.SEMANTIC:\n return performSemanticEvaluation(request);\n default:\n console.warn(\n `Unknown matching approach: ${request.evaluationParameters.approach}, falling back to exact matching`,\n );\n return performEvaluation(request);\n }\n }\n\n private getSafeErrorMessage(error: unknown): string {\n return error instanceof Error ? error.message : 'Field evaluation failed.';\n }\n}\n"]}

package/dist/collection/lib/evaluation/evaluation-service.js CHANGED Viewed

@@ -1,5 +1,6 @@
 import { LLMEvaluationEngine } from "./evaluation-engine";
 import { normalizeEvaluationParametersForField } from "./field-evaluation-approach";
+import { resolveActualValue } from "./actual-value-resolver";
 /**
  * Service for evaluating test case responses
  */
@@ -13,34 +14,71 @@ export class EvaluationService {
      * @param testCase - The test case to evaluate
      * @param onResult - Callback to handle the evaluation result
      */
-    async evaluateTestCase(testCase, onResult) {
-        if (!testCase.output) {
-            console.warn('⚠️ No output to evaluate for test case:', testCase.id);
-            return;
-        }
-        const fields = (testCase.expectedOutcome || []).flatMap((field, index) => {
+    async evaluateTestCase(testCase, onResult, extractors) {
+        const fields = [];
+        const failedFields = [];
+        for (const [index, field] of (testCase.expectedOutcome || []).entries()) {
             if (field.type === 'textarea' && field.outcomeMode === 'dynamic') {
-                return [];
+                continue;
             }
-            return [
-                {
+            const evaluationParameters = normalizeEvaluationParametersForField(field.type, field.evaluationParameters);
+            const expectedValue = getFieldExpectedValue(field);
+            const resolvedActualValue = await resolveActualValue(field, testCase.output, extractors);
+            if (resolvedActualValue.success) {
+                fields.push({
                     index,
                     label: field.label,
                     type: field.type,
-                    expectedValue: getFieldExpectedValue(field),
-                    evaluationParameters: normalizeEvaluationParametersForField(field.type, field.evaluationParameters),
-                },
-            ];
-        });
+                    expectedValue,
+                    actualResponse: resolvedActualValue.value,
+                    evaluationParameters,
+                });
+            }
+            else {
+                failedFields.push({
+                    index,
+                    label: field.label,
+                    type: field.type,
+                    expectedValue,
+                    passed: false,
+                    keywordMatches: [],
+                    evaluationParameters,
+                    evaluationApproachResult: {
+                        score: 0,
+                        approachUsed: evaluationParameters.approach,
+                    },
+                    error: 'error' in resolvedActualValue
+                        ? resolvedActualValue.error
+                        : 'Failed to resolve actual value.',
+                });
+            }
+        }
+        if (fields.length === 0) {
+            if (failedFields.length === 0) {
+                console.warn('⚠️ No evaluable fields for test case:', testCase.id);
+                return;
+            }
+            onResult({
+                testCaseId: testCase.id,
+                passed: false,
+                keywordMatches: [],
+                fieldResults: failedFields,
+                timestamp: new Date().toISOString(),
+            });
+            return;
+        }
         const evaluationRequest = {
             testCaseId: testCase.id,
             question: testCase.question,
-            actualResponse: testCase.output,
             fields,
         };
         await this.engine.evaluateResponse(evaluationRequest, (result) => {
-            console.log('📊 Evaluation result received:', result);
-            onResult(result);
+            const combinedResults = [...(result.fieldResults || []), ...failedFields].sort((a, b) => a.index - b.index);
+            onResult({
+                ...result,
+                passed: combinedResults.every(field => field.passed && !field.error),
+                fieldResults: combinedResults,
+            });
         });
     }
 }

package/dist/collection/lib/evaluation/evaluation-service.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"evaluation-service.js","sourceRoot":"","sources":["../../../src/lib/evaluation/evaluation-service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;~~AAO1D~~,OAAO,EAAE,qCAAqC,EAAE,MAAM,6BAA6B,CAAC;~~AAEpF~~;;GAEG;AACH,MAAM,OAAO,iBAAiB;IACpB,MAAM,CAAsB;IAEpC;QACE,IAAI,CAAC,MAAM,GAAG,IAAI,mBAAmB,EAAE,CAAC;IAC1C,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB,CACpB,QAAkB,EAClB,QAA4C;~~QAE5C~~,~~IAAI~~,~~CAAC~~,~~QAAQ~~,CAAC,MAAM,EAAE,CAAC;~~YACrB~~,~~OAAO~~,CAAC,IAAI,CAAC,~~yCAAyC~~,EAAE,~~QAAQ~~,CAAC,EAAE,CAAC,CAAC;~~YACrE~~,~~OAAO~~;~~QACT~~,CAAC;~~QAED~~,MAAM,~~MAAM~~,~~GAA2B~~,~~CAAC~~,~~QAAQ~~,CAAC,~~eAAe,~~IAAI,~~EAAE~~,CAAC,CAAC,~~OAAO~~,~~CAC7E~~,CAAC,KAAK,~~EAAE~~,KAAK,~~EAAE~~,~~EAAE~~;~~YACf~~,IAAI,~~KAAK~~,CAAC,IAAI,KAAK,~~UAAU~~,~~IAAI~~,KAAK,CAAC,~~WAAW,~~KAAK,~~SAAS~~,EAAE,CAAC;~~gBACjE~~,~~OAAO~~,EAAE,CAAC;~~YACZ~~,CAAC;~~YAED~~,~~OAAO~~;~~gBACL~~;~~oBACE~~,KAAK;oBACL,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,aAAa,EAAE,~~qBAAqB~~,~~CAAC~~,KAAK,CAAC;~~oBAC3C~~,~~oBAAoB~~,EAAE,~~qCAAqC~~,~~CACzD~~,KAAK,~~CAAC~~,IAAI,~~EACV~~,KAAK,CAAC,~~oBAAoB~~,~~CAC3B~~;~~iBACF~~;~~aACF~~,CAAC;~~QACJ~~,CAAC,~~CACF~~,CAAC;~~QAEF~~,MAAM,~~iBAAiB~~,~~GAAwB~~;~~YAC7C~~,~~UAAU~~,EAAE,QAAQ,CAAC,EAAE;~~YACvB~~,QAAQ,EAAE,QAAQ,CAAC,~~QAAQ~~;~~YAC3B~~,cAAc,EAAE,~~QAAQ~~,CAAC,MAAM;~~YAC/B~~,MAAM;SACP,CAAC;QAEF,MAAM,IAAI,CAAC,MAAM,CAAC,gBAAgB,~~CAChC~~,iBAAiB,~~EACjB~~,CAAC,MAAwB,EAAE,EAAE;~~YAC3B~~,~~OAAO~~,CAAC,GAAG,CAAC,~~gCAAgC~~,EAAE,~~MAAM~~,CAAC,CAAC;~~YACtD~~,QAAQ,CAAC,MAAM,CAAC,CAAC;~~QACnB~~,CAAC,~~CACF~~,CAAC;~~IACJ~~,CAAC;CACF;AAED,SAAS,qBAAqB,CAAC,KAA2B;IACxD,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;QACjC,OAAO,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,KAAK,CAAC,KAAK,CAAC;AACrB,CAAC","sourcesContent":["import { LLMEvaluationEngine } from './evaluation-engine';\nimport {\n EvaluationResult,\n FieldEvaluationInput,\n EvaluationRequestV2,\n} from './types';\nimport { TestCase, ExpectedOutcomeField } from '../../types/llm-test-runner';\nimport { normalizeEvaluationParametersForField } from './field-evaluation-approach';\n\n/*\n Service for evaluating test case responses\n /\nexport class EvaluationService {\n private engine: LLMEvaluationEngine;\n\n constructor() {\n this.engine = new LLMEvaluationEngine();\n }\n\n /\n Evaluates a test case response\n * @param testCase - The test case to evaluate\n * @param onResult - Callback to handle the evaluation result\n */\n async evaluateTestCase(\n testCase: TestCase,\n onResult: (result: EvaluationResult) => void,\n ): Promise<void> {\n if ~~(!testCase.output)~~ ~~{\n console.warn('⚠️~~ No ~~output~~ to ~~evaluate~~ ~~for~~ ~~test case:', testCase.id)~~;\n ~~return;\n }~~\n\n const ~~fields:~~ ~~FieldEvaluationInput[~~] = (testCase.expectedOutcome \|\| []).~~flatMap~~(~~\n (field, index~~) => {\n if (field.type === 'textarea' && field.outcomeMode === 'dynamic') {\n ~~return []~~;\n }\n\n ~~return~~ [\n {\n index,\n label: field.label,\n type: field.type,\n expectedValue: ~~getFieldExpectedValue(field)~~,\n evaluationParameters: ~~normalizeEvaluationParametersForField~~(\n field.~~type~~,\n field.evaluationParameters,\n ),\n },\n ];\n },\n );\n\n const evaluationRequest: EvaluationRequestV2 = {\n testCaseId: testCase.id,\n question: testCase.question,\n ~~actualResponse: testCase.output,\n~~ fields,\n };\n\n await this.engine.evaluateResponse(\n evaluationRequest~~,\n~~ (result: EvaluationResult) => {\n ~~console.log('📊~~ ~~Evaluation~~ result ~~received:'~~, ~~result~~);\n onResult(result);\n },\n );\n }\n}\n\nfunction getFieldExpectedValue(field: ExpectedOutcomeField): string {\n if (field.type === 'chips-input') {\n return field.value.join(', ');\n }\n return field.value;\n}\n"]}
1	+ {"version":3,"file":"evaluation-service.js","sourceRoot":"","sources":["../../../src/lib/evaluation/evaluation-service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAY1D,OAAO,EAAE,qCAAqC,EAAE,MAAM,6BAA6B,CAAC;AACpF,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAE7D;;GAEG;AACH,MAAM,OAAO,iBAAiB;IACpB,MAAM,CAAsB;IAEpC;QACE,IAAI,CAAC,MAAM,GAAG,IAAI,mBAAmB,EAAE,CAAC;IAC1C,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB,CACpB,QAAkB,EAClB,QAA4C,EAC5C,UAAuC;QAEvC,MAAM,MAAM,GAA2B,EAAE,CAAC;QAC1C,MAAM,YAAY,GAA4B,EAAE,CAAC;QAEjD,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;YACxE,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;gBACjE,SAAS;YACX,CAAC;YAED,MAAM,oBAAoB,GAAG,qCAAqC,CAChE,KAAK,CAAC,IAAI,EACV,KAAK,CAAC,oBAAoB,CAC3B,CAAC;YACF,MAAM,aAAa,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;YACnD,MAAM,mBAAmB,GAAG,MAAM,kBAAkB,CAClD,KAAK,EACL,QAAQ,CAAC,MAAM,EACf,UAAU,CACX,CAAC;YAEF,IAAI,mBAAmB,CAAC,OAAO,EAAE,CAAC;gBAChC,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK;oBACL,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,aAAa;oBACb,cAAc,EAAE,mBAAmB,CAAC,KAAK;oBACzC,oBAAoB;iBACrB,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,YAAY,CAAC,IAAI,CAAC;oBAChB,KAAK;oBACL,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,aAAa;oBACb,MAAM,EAAE,KAAK;oBACb,cAAc,EAAE,EAAE;oBAClB,oBAAoB;oBACpB,wBAAwB,EAAE;wBACxB,KAAK,EAAE,CAAC;wBACR,YAAY,EAAE,oBAAoB,CAAC,QAAQ;qBAC5C;oBACD,KAAK,EACH,OAAO,IAAI,mBAAmB;wBAC5B,CAAC,CAAC,mBAAmB,CAAC,KAAK;wBAC3B,CAAC,CAAC,iCAAiC;iBACxC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9B,OAAO,CAAC,IAAI,CAAC,uCAAuC,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACnE,OAAO;YACT,CAAC;YAED,QAAQ,CAAC;gBACP,UAAU,EAAE,QAAQ,CAAC,EAAE;gBACvB,MAAM,EAAE,KAAK;gBACb,cAAc,EAAE,EAAE;gBAClB,YAAY,EAAE,YAAY;gBAC1B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,MAAM,iBAAiB,GAAwB;YAC7C,UAAU,EAAE,QAAQ,CAAC,EAAE;YACvB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,MAAM;SACP,CAAC;QAEF,MAAM,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,CAAC,MAAwB,EAAE,EAAE;YACjF,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,CAAC,EAAE,GAAG,YAAY,CAAC,CAAC,IAAI,CAC5E,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAC5B,CAAC;YACF,QAAQ,CAAC;gBACP,GAAG,MAAM;gBACT,MAAM,EAAE,eAAe,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;gBACpE,YAAY,EAAE,eAAe;aAC9B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AAED,SAAS,qBAAqB,CAAC,KAA2B;IACxD,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;QACjC,OAAO,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,KAAK,CAAC,KAAK,CAAC;AACrB,CAAC","sourcesContent":["import { LLMEvaluationEngine } from './evaluation-engine';\nimport {\n EvaluationResult,\n FieldEvaluationInput,\n EvaluationRequestV2,\n FieldEvaluationResult,\n} from './types';\nimport {\n TestCase,\n ExpectedOutcomeField,\n EvaluationSourceExtractors,\n} from '../../types/llm-test-runner';\nimport { normalizeEvaluationParametersForField } from './field-evaluation-approach';\nimport { resolveActualValue } from './actual-value-resolver';\n\n/*\n Service for evaluating test case responses\n /\nexport class EvaluationService {\n private engine: LLMEvaluationEngine;\n\n constructor() {\n this.engine = new LLMEvaluationEngine();\n }\n\n /\n Evaluates a test case response\n * @param testCase - The test case to evaluate\n * @param onResult - Callback to handle the evaluation result\n */\n async evaluateTestCase(\n testCase: TestCase,\n onResult: (result: EvaluationResult) => void,\n extractors?: EvaluationSourceExtractors,\n ): Promise<void> {\n const fields: FieldEvaluationInput[] = [];\n const failedFields: FieldEvaluationResult[] = [];\n\n for (const [index, field] of (testCase.expectedOutcome \|\| []).entries()) {\n if (field.type === 'textarea' && field.outcomeMode === 'dynamic') {\n continue;\n }\n\n const evaluationParameters = normalizeEvaluationParametersForField(\n field.type,\n field.evaluationParameters,\n );\n const expectedValue = getFieldExpectedValue(field);\n const resolvedActualValue = await resolveActualValue(\n field,\n testCase.output,\n extractors,\n );\n\n if (resolvedActualValue.success) {\n fields.push({\n index,\n label: field.label,\n type: field.type,\n expectedValue,\n actualResponse: resolvedActualValue.value,\n evaluationParameters,\n });\n } else {\n failedFields.push({\n index,\n label: field.label,\n type: field.type,\n expectedValue,\n passed: false,\n keywordMatches: [],\n evaluationParameters,\n evaluationApproachResult: {\n score: 0,\n approachUsed: evaluationParameters.approach,\n },\n error:\n 'error' in resolvedActualValue\n ? resolvedActualValue.error\n : 'Failed to resolve actual value.',\n });\n }\n }\n\n if (fields.length === 0) {\n if (failedFields.length === 0) {\n console.warn('⚠️ No evaluable fields for test case:', testCase.id);\n return;\n }\n\n onResult({\n testCaseId: testCase.id,\n passed: false,\n keywordMatches: [],\n fieldResults: failedFields,\n timestamp: new Date().toISOString(),\n });\n return;\n }\n\n const evaluationRequest: EvaluationRequestV2 = {\n testCaseId: testCase.id,\n question: testCase.question,\n fields,\n };\n\n await this.engine.evaluateResponse(evaluationRequest, (result: EvaluationResult) => {\n const combinedResults = [...(result.fieldResults \|\| []), ...failedFields].sort(\n (a, b) => a.index - b.index,\n );\n onResult({\n ...result,\n passed: combinedResults.every(field => field.passed && !field.error),\n fieldResults: combinedResults,\n });\n });\n }\n}\n\nfunction getFieldExpectedValue(field: ExpectedOutcomeField): string {\n if (field.type === 'chips-input') {\n return field.value.join(', ');\n }\n return field.value;\n}\n"]}

package/dist/collection/lib/evaluation/types.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/lib/evaluation/types.ts"],"names":[],"mappings":"","sourcesContent":["import {\n EvaluationParameters,\n EvaluationApproachResult,\n} from '../../types/evaluation';\nimport type { ExpectedOutcomeFieldType } from '../../types/llm-test-runner';\n\nexport interface EvaluationRequest {\n testCaseId: string;\n question: string;\n expectedOutcome: string;\n actualResponse: string;\n evaluationParameters: EvaluationParameters;\n}\n\nexport interface FieldEvaluationInput {\n index: number;\n label: string;\n type: ExpectedOutcomeFieldType;\n expectedValue: string;\n evaluationParameters: EvaluationParameters;\n}\n\nexport interface EvaluationRequestV2 {\n testCaseId: string;\n question: string;\n ~~actualResponse: string;\n~~ fields: FieldEvaluationInput[];\n}\n\nexport interface EvaluationResult {\n testCaseId: string;\n passed: boolean;\n keywordMatches: KeywordMatch[];\n fieldResults?: FieldEvaluationResult[];\n timestamp?: string;\n evaluationParameters?: EvaluationParameters;\n evaluationApproachResult?: EvaluationApproachResult;\n}\n\nexport interface FieldEvaluationResult {\n index: number;\n label: string;\n type: ExpectedOutcomeFieldType;\n expectedValue: string;\n passed: boolean;\n keywordMatches: KeywordMatch[];\n evaluationParameters: EvaluationParameters;\n evaluationApproachResult: EvaluationApproachResult;\n error?: string;\n}\n\nexport interface KeywordMatch {\n keyword: string;\n found: boolean;\n evaluationApproachResult: EvaluationApproachResult;\n}\n\nexport type EvaluationCallback = (result: EvaluationResult) => void;\n\nexport interface RougeKeywordDetails {\n rouge1: number;\n rougeL: number;\n scoreUsed: string;\n approach: string;\n}\n\nexport interface Rouge1OverallDetails {\n keywordsPassed: number;\n totalKeywords: number;\n passRate: string;\n thresholdUsed: number;\n approach: string;\n}\n"]}
1	+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/lib/evaluation/types.ts"],"names":[],"mappings":"","sourcesContent":["import {\n EvaluationParameters,\n EvaluationApproachResult,\n} from '../../types/evaluation';\nimport type { ExpectedOutcomeFieldType } from '../../types/llm-test-runner';\n\nexport interface EvaluationRequest {\n testCaseId: string;\n question: string;\n expectedOutcome: string;\n actualResponse: string;\n evaluationParameters: EvaluationParameters;\n}\n\nexport interface FieldEvaluationInput {\n index: number;\n label: string;\n type: ExpectedOutcomeFieldType;\n expectedValue: string;\n actualResponse: string;\n evaluationParameters: EvaluationParameters;\n}\n\nexport interface EvaluationRequestV2 {\n testCaseId: string;\n question: string;\n fields: FieldEvaluationInput[];\n}\n\nexport interface EvaluationResult {\n testCaseId: string;\n passed: boolean;\n keywordMatches: KeywordMatch[];\n fieldResults?: FieldEvaluationResult[];\n timestamp?: string;\n evaluationParameters?: EvaluationParameters;\n evaluationApproachResult?: EvaluationApproachResult;\n}\n\nexport interface FieldEvaluationResult {\n index: number;\n label: string;\n type: ExpectedOutcomeFieldType;\n expectedValue: string;\n passed: boolean;\n keywordMatches: KeywordMatch[];\n evaluationParameters: EvaluationParameters;\n evaluationApproachResult: EvaluationApproachResult;\n error?: string;\n}\n\nexport interface KeywordMatch {\n keyword: string;\n found: boolean;\n evaluationApproachResult: EvaluationApproachResult;\n}\n\nexport type EvaluationCallback = (result: EvaluationResult) => void;\n\nexport interface RougeKeywordDetails {\n rouge1: number;\n rougeL: number;\n scoreUsed: string;\n approach: string;\n}\n\nexport interface Rouge1OverallDetails {\n keywordsPassed: number;\n totalKeywords: number;\n passRate: string;\n thresholdUsed: number;\n approach: string;\n}\n"]}

package/dist/collection/lib/form/components/app-textarea.css CHANGED Viewed

@@ -38,8 +38,8 @@
   background: var(--muted);
   color: var(--muted-foreground);
   border-color: var(--border);
-  cursor: not-allowed;
-  resize: none;
+  cursor: default;
+  resize: vertical;
 }
 .textarea-element:read-only:focus {

package/dist/collection/lib/import-export/test-suite-importer.js CHANGED Viewed

@@ -1,14 +1,20 @@
 import { createTestCaseFromInput } from "../test-cases/test-case-factory";
 import { validateTestCaseInputArray } from "../../schemas/test-case";
+import { validateExpectedOutcomeArrayWithExtractors } from "../../schemas/expected-outcome";
 /**
  * Validates and imports test cases from JSON content
  * @param jsonContent - The JSON string to parse and validate
  * @returns Validation result with test cases or error message
  */
-export function importTestSuite(jsonContent) {
+export function importTestSuite(jsonContent, allowedExtractorIds = []) {
     try {
         const parsed = JSON.parse(jsonContent);
         validateTestCaseInputArray(parsed);
+        if (allowedExtractorIds.length > 0) {
+            parsed.forEach((testCase) => {
+                validateExpectedOutcomeArrayWithExtractors(testCase.expectedOutcome, allowedExtractorIds);
+            });
+        }
         const testCases = parsed.map((item, index) => {
             try {
                 return createTestCaseFromInput(item);

package/dist/collection/lib/import-export/test-suite-importer.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"test-suite-importer.js","sourceRoot":"","sources":["../../../src/lib/import-export/test-suite-importer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAC1E,OAAO,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;~~AAQrE~~;;;;GAIG;AACH,MAAM,UAAU,eAAe,~~CAAC~~,WAAmB;~~IACjD~~,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACvC,0BAA0B,CAAC,MAAM,CAAC,CAAC;~~QAEnC~~,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YAC3C,IAAI,CAAC;gBACH,OAAO,uBAAuB,CAAC,IAAI,CAAC,CAAC;YACvC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;gBACrE,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,KAAK,OAAO,EAAE,CAAC,CAAC;YACrE,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,SAAS;SACV,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EACH,GAAG,YAAY,KAAK;gBAClB,CAAC,CAAC,GAAG,CAAC,OAAO;gBACb,CAAC,CAAC,gEAAgE;SACvE,CAAC;IACJ,CAAC;AACH,CAAC","sourcesContent":["import type { TestCase } from '../../types/llm-test-runner';\nimport { createTestCaseFromInput } from '../test-cases/test-case-factory';\nimport { validateTestCaseInputArray } from '../../schemas/test-case';\n\nexport interface ImportValidationResult {\n success: boolean;\n testCases?: TestCase[];\n error?: string;\n}\n\n/*\n Validates and imports test cases from JSON content\n * @param jsonContent - The JSON string to parse and validate\n * @returns Validation result with test cases or error message\n */\nexport function importTestSuite(jsonContent: string): ImportValidationResult {\n try {\n const parsed = JSON.parse(jsonContent);\n validateTestCaseInputArray(parsed);\n\n const testCases = parsed.map((item, index) => {\n try {\n return createTestCaseFromInput(item);\n } catch (err) {\n const message = err instanceof Error ? err.message : 'Unknown error';\n throw new Error(`Invalid test case at index ${index}: ${message}`);\n }\n });\n\n return {\n success: true,\n testCases,\n };\n } catch (err) {\n return {\n success: false,\n error:\n err instanceof Error\n ? err.message\n : 'Error processing file. Please ensure it is a valid JSON array.',\n };\n }\n}\n\n"]}
1	+ {"version":3,"file":"test-suite-importer.js","sourceRoot":"","sources":["../../../src/lib/import-export/test-suite-importer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAC1E,OAAO,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;AACrE,OAAO,EAAE,0CAA0C,EAAE,MAAM,gCAAgC,CAAC;AAQ5F;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAC7B,WAAmB,EACnB,sBAAgC,EAAE;IAElC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACvC,0BAA0B,CAAC,MAAM,CAAC,CAAC;QACnC,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;gBAC1B,0CAA0C,CACxC,QAAQ,CAAC,eAAe,EACxB,mBAAmB,CACpB,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YAC3C,IAAI,CAAC;gBACH,OAAO,uBAAuB,CAAC,IAAI,CAAC,CAAC;YACvC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;gBACrE,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,KAAK,OAAO,EAAE,CAAC,CAAC;YACrE,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,SAAS;SACV,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EACH,GAAG,YAAY,KAAK;gBAClB,CAAC,CAAC,GAAG,CAAC,OAAO;gBACb,CAAC,CAAC,gEAAgE;SACvE,CAAC;IACJ,CAAC;AACH,CAAC","sourcesContent":["import type { TestCase } from '../../types/llm-test-runner';\nimport { createTestCaseFromInput } from '../test-cases/test-case-factory';\nimport { validateTestCaseInputArray } from '../../schemas/test-case';\nimport { validateExpectedOutcomeArrayWithExtractors } from '../../schemas/expected-outcome';\n\nexport interface ImportValidationResult {\n success: boolean;\n testCases?: TestCase[];\n error?: string;\n}\n\n/*\n Validates and imports test cases from JSON content\n * @param jsonContent - The JSON string to parse and validate\n * @returns Validation result with test cases or error message\n */\nexport function importTestSuite(\n jsonContent: string,\n allowedExtractorIds: string[] = [],\n): ImportValidationResult {\n try {\n const parsed = JSON.parse(jsonContent);\n validateTestCaseInputArray(parsed);\n if (allowedExtractorIds.length > 0) {\n parsed.forEach((testCase) => {\n validateExpectedOutcomeArrayWithExtractors(\n testCase.expectedOutcome,\n allowedExtractorIds,\n );\n });\n }\n\n const testCases = parsed.map((item, index) => {\n try {\n return createTestCaseFromInput(item);\n } catch (err) {\n const message = err instanceof Error ? err.message : 'Unknown error';\n throw new Error(`Invalid test case at index ${index}: ${message}`);\n }\n });\n\n return {\n success: true,\n testCases,\n };\n } catch (err) {\n return {\n success: false,\n error:\n err instanceof Error\n ? err.message\n : 'Error processing file. Please ensure it is a valid JSON array.',\n };\n }\n}\n\n"]}

package/dist/collection/lib/test-cases/test-case-factory.js CHANGED Viewed

@@ -11,6 +11,7 @@ export const DEFAULT_EXPECTED_OUTCOME_SCHEMA = [
 function normalizeExpectedOutcomeField(field) {
     return {
         ...field,
+        evaluationSource: field.evaluationSource || { type: 'text' },
         evaluationParameters: normalizeEvaluationParametersForField(field.type, field.evaluationParameters),
     };
 }
@@ -34,6 +35,7 @@ function createExpectedOutcomeFieldFromSchema(schemaField) {
                 type: 'text',
                 label: schemaField.label,
                 placeholder: schemaField.placeholder,
+                evaluationSource: schemaField.evaluationSource || { type: 'text' },
                 value: '',
                 evaluationParameters: normalizeEvaluationParametersForField(schemaField.type, schemaField.evaluationParameters),
             };
@@ -42,6 +44,7 @@ function createExpectedOutcomeFieldFromSchema(schemaField) {
                 type: 'textarea',
                 label: schemaField.label,
                 placeholder: schemaField.placeholder,
+                evaluationSource: schemaField.evaluationSource || { type: 'text' },
                 rows: schemaField.rows,
                 value: '',
                 evaluationParameters: normalizeEvaluationParametersForField(schemaField.type, schemaField.evaluationParameters),
@@ -51,6 +54,7 @@ function createExpectedOutcomeFieldFromSchema(schemaField) {
                 type: 'chips-input',
                 label: schemaField.label,
                 placeholder: schemaField.placeholder,
+                evaluationSource: schemaField.evaluationSource || { type: 'text' },
                 value: [],
                 evaluationParameters: normalizeEvaluationParametersForField(schemaField.type, schemaField.evaluationParameters),
             };
@@ -59,6 +63,7 @@ function createExpectedOutcomeFieldFromSchema(schemaField) {
                 type: 'select',
                 label: schemaField.label,
                 placeholder: schemaField.placeholder,
+                evaluationSource: schemaField.evaluationSource || { type: 'text' },
                 value: schemaField.options[0],
                 options: schemaField.options,
                 evaluationParameters: normalizeEvaluationParametersForField(schemaField.type, schemaField.evaluationParameters),