npm - llm-testrunner-components - Versions diffs - 1.2.3 → 1.3.0 - Mend

llm-testrunner-components 1.2.3 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (110) hide show

package/dist/collection/lib/evaluation/evaluation-engine.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"evaluation-engine.js","sourceRoot":"","sources":["../../../src/lib/evaluation/evaluation-engine.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AACxE,OAAO,EAAE,yBAAyB,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AACxE,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AAEzE,MAAM,OAAO,mBAAmB;IAC9B,KAAK,CAAC,gBAAgB,CACpB,OAA4B,EAC5B,QAA4B;QAE5B,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,UAAU,CAC7C,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAC,KAAK,EAAC,EAAE;YAC/B,MAAM,YAAY,GAAsB;gBACtC,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,cAAc,EAAE,~~OAAO~~,CAAC,cAAc;~~gBACtC~~,eAAe,EAAE,KAAK,CAAC,aAAa;gBACpC,oBAAoB,EAAE,KAAK,CAAC,oBAAoB;aACjD,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC;YAEtD,MAAM,WAAW,GAA0B;gBACzC,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,aAAa,EAAE,KAAK,CAAC,aAAa;gBAClC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,oBAAoB,EAAE,MAAM,CAAC,oBAAqB;gBAClD,wBAAwB,EAAE,MAAM,CAAC,wBAAwB;aAC1D,CAAC;YACF,OAAO,WAAW,CAAC;QACrB,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,YAAY,GAA4B,cAAc,CAAC,GAAG,CAC9D,CAAC,aAAa,EAAE,KAAK,EAAE,EAAE;YACvB,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACpC,IAAI,aAAa,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBACzC,OAAO,aAAa,CAAC,KAAK,CAAC;YAC7B,CAAC;YAED,OAAO;gBACL,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,aAAa,EAAE,KAAK,CAAC,aAAa;gBAClC,MAAM,EAAE,KAAK;gBACb,cAAc,EAAE,EAAE;gBAClB,oBAAoB,EAAE,KAAK,CAAC,oBAAoB;gBAChD,wBAAwB,EAAE;oBACxB,KAAK,EAAE,CAAC;oBACR,YAAY,EAAE,KAAK,CAAC,oBAAoB,CAAC,QAAQ;iBAClD;gBACD,KAAK,EAAE,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,MAAM,CAAC;aACtD,CAAC;QACJ,CAAC,CACF,CAAC;QAEF,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAC3E,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEzE,QAAQ,CAAC;YACP,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,MAAM;YACN,cAAc;YACd,YAAY;YACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAA0B;QACpD,MAAM,QAAQ,GAAuB,OAAO,CAAC,oBAAoB,CAAC,QAAQ,CAAC;QAC3E,QAAQ,QAAQ,EAAE,CAAC;YACjB,KAAK,kBAAkB,CAAC,IAAI;gBAC1B,OAAO,qBAAqB,CAAC,OAAO,CAAC,CAAC;YACxC,KAAK,kBAAkB,CAAC,KAAK;gBAC3B,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;YACpC,KAAK,kBAAkB,CAAC,OAAO;gBAC7B,OAAO,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAC1C,KAAK,kBAAkB,CAAC,OAAO;gBAC7B,OAAO,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAC1C,KAAK,kBAAkB,CAAC,QAAQ;gBAC9B,OAAO,yBAAyB,CAAC,OAAO,CAAC,CAAC;YAC5C;gBACE,OAAO,CAAC,IAAI,CACV,8BAA8B,OAAO,CAAC,oBAAoB,CAAC,QAAQ,kCAAkC,CACtG,CAAC;gBACF,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAEO,mBAAmB,CAAC,KAAc;QACxC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,0BAA0B,CAAC;IAC7E,CAAC;CACF","sourcesContent":["import {\n EvaluationRequest,\n EvaluationResult,\n EvaluationCallback,\n FieldEvaluationResult,\n EvaluationRequestV2,\n} from './types';\nimport { performEvaluation } from './evaluators/exact/exact';\nimport { EvaluationApproach } from './constants';\nimport { performRouge1Evaluation } from './evaluators/rouge1-evaluator';\nimport { performSemanticEvaluation } from './evaluators/semantic/index';\nimport { performRougeLEvaluation } from './evaluators/rougeL-evaluator';\nimport { performBleuEvaluation } from './evaluators/bleu/bleu-evaluator';\n\nexport class LLMEvaluationEngine {\n async evaluateResponse(\n request: EvaluationRequestV2,\n callback: EvaluationCallback,\n ): Promise<void> {\n const settledResults = await Promise.allSettled(\n request.fields.map(async field => {\n const fieldRequest: EvaluationRequest = {\n testCaseId: request.testCaseId,\n question: request.question,\n actualResponse: ~~request~~.actualResponse,\n expectedOutcome: field.expectedValue,\n evaluationParameters: field.evaluationParameters,\n };\n const result = await this.evaluateField(fieldRequest);\n\n const fieldResult: FieldEvaluationResult = {\n index: field.index,\n label: field.label,\n type: field.type,\n expectedValue: field.expectedValue,\n passed: result.passed,\n keywordMatches: result.keywordMatches,\n evaluationParameters: result.evaluationParameters!,\n evaluationApproachResult: result.evaluationApproachResult,\n };\n return fieldResult;\n }),\n );\n\n const fieldResults: FieldEvaluationResult[] = settledResults.map(\n (settledResult, index) => {\n const field = request.fields[index];\n if (settledResult.status === 'fulfilled') {\n return settledResult.value;\n }\n\n return {\n index: field.index,\n label: field.label,\n type: field.type,\n expectedValue: field.expectedValue,\n passed: false,\n keywordMatches: [],\n evaluationParameters: field.evaluationParameters,\n evaluationApproachResult: {\n score: 0,\n approachUsed: field.evaluationParameters.approach,\n },\n error: this.getSafeErrorMessage(settledResult.reason),\n };\n },\n );\n\n const keywordMatches = fieldResults.flatMap(field => field.keywordMatches);\n const passed = fieldResults.every(field => field.passed && !field.error);\n\n callback({\n testCaseId: request.testCaseId,\n passed,\n keywordMatches,\n fieldResults,\n timestamp: new Date().toISOString(),\n });\n }\n\n private async evaluateField(request: EvaluationRequest): Promise<EvaluationResult> {\n const approach: EvaluationApproach = request.evaluationParameters.approach;\n switch (approach) {\n case EvaluationApproach.BLEU:\n return performBleuEvaluation(request);\n case EvaluationApproach.EXACT:\n return performEvaluation(request);\n case EvaluationApproach.ROUGE_1:\n return performRouge1Evaluation(request);\n case EvaluationApproach.ROUGE_L:\n return performRougeLEvaluation(request);\n case EvaluationApproach.SEMANTIC:\n return performSemanticEvaluation(request);\n default:\n console.warn(\n `Unknown matching approach: ${request.evaluationParameters.approach}, falling back to exact matching`,\n );\n return performEvaluation(request);\n }\n }\n\n private getSafeErrorMessage(error: unknown): string {\n return error instanceof Error ? error.message : 'Field evaluation failed.';\n }\n}\n"]}
1	+ {"version":3,"file":"evaluation-engine.js","sourceRoot":"","sources":["../../../src/lib/evaluation/evaluation-engine.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AACxE,OAAO,EAAE,yBAAyB,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AACxE,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AAEzE,MAAM,OAAO,mBAAmB;IAC9B,KAAK,CAAC,gBAAgB,CACpB,OAA4B,EAC5B,QAA4B;QAE5B,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,UAAU,CAC7C,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAC,KAAK,EAAC,EAAE;YAC/B,MAAM,YAAY,GAAsB;gBACtC,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,cAAc,EAAE,KAAK,CAAC,cAAc;gBACpC,eAAe,EAAE,KAAK,CAAC,aAAa;gBACpC,oBAAoB,EAAE,KAAK,CAAC,oBAAoB;aACjD,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC;YAEtD,MAAM,WAAW,GAA0B;gBACzC,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,aAAa,EAAE,KAAK,CAAC,aAAa;gBAClC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,oBAAoB,EAAE,MAAM,CAAC,oBAAqB;gBAClD,wBAAwB,EAAE,MAAM,CAAC,wBAAwB;aAC1D,CAAC;YACF,OAAO,WAAW,CAAC;QACrB,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,YAAY,GAA4B,cAAc,CAAC,GAAG,CAC9D,CAAC,aAAa,EAAE,KAAK,EAAE,EAAE;YACvB,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACpC,IAAI,aAAa,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBACzC,OAAO,aAAa,CAAC,KAAK,CAAC;YAC7B,CAAC;YAED,OAAO;gBACL,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,aAAa,EAAE,KAAK,CAAC,aAAa;gBAClC,MAAM,EAAE,KAAK;gBACb,cAAc,EAAE,EAAE;gBAClB,oBAAoB,EAAE,KAAK,CAAC,oBAAoB;gBAChD,wBAAwB,EAAE;oBACxB,KAAK,EAAE,CAAC;oBACR,YAAY,EAAE,KAAK,CAAC,oBAAoB,CAAC,QAAQ;iBAClD;gBACD,KAAK,EAAE,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,MAAM,CAAC;aACtD,CAAC;QACJ,CAAC,CACF,CAAC;QAEF,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAC3E,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEzE,QAAQ,CAAC;YACP,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,MAAM;YACN,cAAc;YACd,YAAY;YACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAA0B;QACpD,MAAM,QAAQ,GAAuB,OAAO,CAAC,oBAAoB,CAAC,QAAQ,CAAC;QAC3E,QAAQ,QAAQ,EAAE,CAAC;YACjB,KAAK,kBAAkB,CAAC,IAAI;gBAC1B,OAAO,qBAAqB,CAAC,OAAO,CAAC,CAAC;YACxC,KAAK,kBAAkB,CAAC,KAAK;gBAC3B,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;YACpC,KAAK,kBAAkB,CAAC,OAAO;gBAC7B,OAAO,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAC1C,KAAK,kBAAkB,CAAC,OAAO;gBAC7B,OAAO,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAC1C,KAAK,kBAAkB,CAAC,QAAQ;gBAC9B,OAAO,yBAAyB,CAAC,OAAO,CAAC,CAAC;YAC5C;gBACE,OAAO,CAAC,IAAI,CACV,8BAA8B,OAAO,CAAC,oBAAoB,CAAC,QAAQ,kCAAkC,CACtG,CAAC;gBACF,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAEO,mBAAmB,CAAC,KAAc;QACxC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,0BAA0B,CAAC;IAC7E,CAAC;CACF","sourcesContent":["import {\n EvaluationRequest,\n EvaluationResult,\n EvaluationCallback,\n FieldEvaluationResult,\n EvaluationRequestV2,\n} from './types';\nimport { performEvaluation } from './evaluators/exact/exact';\nimport { EvaluationApproach } from './constants';\nimport { performRouge1Evaluation } from './evaluators/rouge1-evaluator';\nimport { performSemanticEvaluation } from './evaluators/semantic/index';\nimport { performRougeLEvaluation } from './evaluators/rougeL-evaluator';\nimport { performBleuEvaluation } from './evaluators/bleu/bleu-evaluator';\n\nexport class LLMEvaluationEngine {\n async evaluateResponse(\n request: EvaluationRequestV2,\n callback: EvaluationCallback,\n ): Promise<void> {\n const settledResults = await Promise.allSettled(\n request.fields.map(async field => {\n const fieldRequest: EvaluationRequest = {\n testCaseId: request.testCaseId,\n question: request.question,\n actualResponse: field.actualResponse,\n expectedOutcome: field.expectedValue,\n evaluationParameters: field.evaluationParameters,\n };\n const result = await this.evaluateField(fieldRequest);\n\n const fieldResult: FieldEvaluationResult = {\n index: field.index,\n label: field.label,\n type: field.type,\n expectedValue: field.expectedValue,\n passed: result.passed,\n keywordMatches: result.keywordMatches,\n evaluationParameters: result.evaluationParameters!,\n evaluationApproachResult: result.evaluationApproachResult,\n };\n return fieldResult;\n }),\n );\n\n const fieldResults: FieldEvaluationResult[] = settledResults.map(\n (settledResult, index) => {\n const field = request.fields[index];\n if (settledResult.status === 'fulfilled') {\n return settledResult.value;\n }\n\n return {\n index: field.index,\n label: field.label,\n type: field.type,\n expectedValue: field.expectedValue,\n passed: false,\n keywordMatches: [],\n evaluationParameters: field.evaluationParameters,\n evaluationApproachResult: {\n score: 0,\n approachUsed: field.evaluationParameters.approach,\n },\n error: this.getSafeErrorMessage(settledResult.reason),\n };\n },\n );\n\n const keywordMatches = fieldResults.flatMap(field => field.keywordMatches);\n const passed = fieldResults.every(field => field.passed && !field.error);\n\n callback({\n testCaseId: request.testCaseId,\n passed,\n keywordMatches,\n fieldResults,\n timestamp: new Date().toISOString(),\n });\n }\n\n private async evaluateField(request: EvaluationRequest): Promise<EvaluationResult> {\n const approach: EvaluationApproach = request.evaluationParameters.approach;\n switch (approach) {\n case EvaluationApproach.BLEU:\n return performBleuEvaluation(request);\n case EvaluationApproach.EXACT:\n return performEvaluation(request);\n case EvaluationApproach.ROUGE_1:\n return performRouge1Evaluation(request);\n case EvaluationApproach.ROUGE_L:\n return performRougeLEvaluation(request);\n case EvaluationApproach.SEMANTIC:\n return performSemanticEvaluation(request);\n default:\n console.warn(\n `Unknown matching approach: ${request.evaluationParameters.approach}, falling back to exact matching`,\n );\n return performEvaluation(request);\n }\n }\n\n private getSafeErrorMessage(error: unknown): string {\n return error instanceof Error ? error.message : 'Field evaluation failed.';\n }\n}\n"]}

package/dist/collection/lib/evaluation/evaluation-service.js CHANGED Viewed

@@ -1,5 +1,6 @@
 import { LLMEvaluationEngine } from "./evaluation-engine";
 import { normalizeEvaluationParametersForField } from "./field-evaluation-approach";
+import { resolveActualValue } from "./actual-value-resolver";
 /**
  * Service for evaluating test case responses
  */
@@ -13,34 +14,71 @@ export class EvaluationService {
      * @param testCase - The test case to evaluate
      * @param onResult - Callback to handle the evaluation result
      */
-    async evaluateTestCase(testCase, onResult) {
-        if (!testCase.output) {
-            console.warn('⚠️ No output to evaluate for test case:', testCase.id);
-            return;
-        }
-        const fields = (testCase.expectedOutcome || []).flatMap((field, index) => {
+    async evaluateTestCase(testCase, onResult, extractors) {
+        const fields = [];
+        const failedFields = [];
+        for (const [index, field] of (testCase.expectedOutcome || []).entries()) {
             if (field.type === 'textarea' && field.outcomeMode === 'dynamic') {
-                return [];
+                continue;
             }
-            return [
-                {
+            const evaluationParameters = normalizeEvaluationParametersForField(field.type, field.evaluationParameters);
+            const expectedValue = getFieldExpectedValue(field);
+            const resolvedActualValue = await resolveActualValue(field, testCase.output, extractors);
+            if (resolvedActualValue.success) {
+                fields.push({
                     index,
                     label: field.label,
                     type: field.type,
-                    expectedValue: getFieldExpectedValue(field),
-                    evaluationParameters: normalizeEvaluationParametersForField(field.type, field.evaluationParameters),
-                },
-            ];
-        });
+                    expectedValue,
+                    actualResponse: resolvedActualValue.value,
+                    evaluationParameters,
+                });
+            }
+            else {
+                failedFields.push({
+                    index,
+                    label: field.label,
+                    type: field.type,
+                    expectedValue,
+                    passed: false,
+                    keywordMatches: [],
+                    evaluationParameters,
+                    evaluationApproachResult: {
+                        score: 0,
+                        approachUsed: evaluationParameters.approach,
+                    },
+                    error: 'error' in resolvedActualValue
+                        ? resolvedActualValue.error
+                        : 'Failed to resolve actual value.',
+                });
+            }
+        }
+        if (fields.length === 0) {
+            if (failedFields.length === 0) {
+                console.warn('⚠️ No evaluable fields for test case:', testCase.id);
+                return;
+            }
+            onResult({
+                testCaseId: testCase.id,
+                passed: false,
+                keywordMatches: [],
+                fieldResults: failedFields,
+                timestamp: new Date().toISOString(),
+            });
+            return;
+        }
         const evaluationRequest = {
             testCaseId: testCase.id,
             question: testCase.question,
-            actualResponse: testCase.output,
             fields,
         };
         await this.engine.evaluateResponse(evaluationRequest, (result) => {
-            console.log('📊 Evaluation result received:', result);
-            onResult(result);
+            const combinedResults = [...(result.fieldResults || []), ...failedFields].sort((a, b) => a.index - b.index);
+            onResult({
+                ...result,
+                passed: combinedResults.every(field => field.passed && !field.error),
+                fieldResults: combinedResults,
+            });
         });
     }
 }

package/dist/collection/lib/evaluation/evaluation-service.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"evaluation-service.js","sourceRoot":"","sources":["../../../src/lib/evaluation/evaluation-service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;~~AAO1D~~,OAAO,EAAE,qCAAqC,EAAE,MAAM,6BAA6B,CAAC;~~AAEpF~~;;GAEG;AACH,MAAM,OAAO,iBAAiB;IACpB,MAAM,CAAsB;IAEpC;QACE,IAAI,CAAC,MAAM,GAAG,IAAI,mBAAmB,EAAE,CAAC;IAC1C,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB,CACpB,QAAkB,EAClB,QAA4C;~~QAE5C~~,~~IAAI~~,~~CAAC~~,~~QAAQ~~,CAAC,MAAM,EAAE,CAAC;~~YACrB~~,~~OAAO~~,CAAC,IAAI,CAAC,~~yCAAyC~~,EAAE,~~QAAQ~~,CAAC,EAAE,CAAC,CAAC;~~YACrE~~,~~OAAO~~;~~QACT~~,CAAC;~~QAED~~,MAAM,~~MAAM~~,~~GAA2B~~,~~CAAC~~,~~QAAQ~~,CAAC,~~eAAe,~~IAAI,~~EAAE~~,CAAC,CAAC,~~OAAO~~,~~CAC7E~~,CAAC,KAAK,~~EAAE~~,KAAK,~~EAAE~~,~~EAAE~~;~~YACf~~,IAAI,~~KAAK~~,CAAC,IAAI,KAAK,~~UAAU~~,~~IAAI~~,KAAK,CAAC,~~WAAW,~~KAAK,~~SAAS~~,EAAE,CAAC;~~gBACjE~~,~~OAAO~~,EAAE,CAAC;~~YACZ~~,CAAC;~~YAED~~,~~OAAO~~;~~gBACL~~;~~oBACE~~,KAAK;oBACL,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,aAAa,EAAE,~~qBAAqB~~,~~CAAC~~,KAAK,CAAC;~~oBAC3C~~,~~oBAAoB~~,EAAE,~~qCAAqC~~,~~CACzD~~,KAAK,~~CAAC~~,IAAI,~~EACV~~,KAAK,CAAC,~~oBAAoB~~,~~CAC3B~~;~~iBACF~~;~~aACF~~,CAAC;~~QACJ~~,CAAC,~~CACF~~,CAAC;~~QAEF~~,MAAM,~~iBAAiB~~,~~GAAwB~~;~~YAC7C~~,~~UAAU~~,EAAE,QAAQ,CAAC,EAAE;~~YACvB~~,QAAQ,EAAE,QAAQ,CAAC,~~QAAQ~~;~~YAC3B~~,cAAc,EAAE,~~QAAQ~~,CAAC,MAAM;~~YAC/B~~,MAAM;SACP,CAAC;QAEF,MAAM,IAAI,CAAC,MAAM,CAAC,gBAAgB,~~CAChC~~,iBAAiB,~~EACjB~~,CAAC,MAAwB,EAAE,EAAE;~~YAC3B~~,~~OAAO~~,CAAC,GAAG,CAAC,~~gCAAgC~~,EAAE,~~MAAM~~,CAAC,CAAC;~~YACtD~~,QAAQ,CAAC,MAAM,CAAC,CAAC;~~QACnB~~,CAAC,~~CACF~~,CAAC;~~IACJ~~,CAAC;CACF;AAED,SAAS,qBAAqB,CAAC,KAA2B;IACxD,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;QACjC,OAAO,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,KAAK,CAAC,KAAK,CAAC;AACrB,CAAC","sourcesContent":["import { LLMEvaluationEngine } from './evaluation-engine';\nimport {\n EvaluationResult,\n FieldEvaluationInput,\n EvaluationRequestV2,\n} from './types';\nimport { TestCase, ExpectedOutcomeField } from '../../types/llm-test-runner';\nimport { normalizeEvaluationParametersForField } from './field-evaluation-approach';\n\n/*\n Service for evaluating test case responses\n /\nexport class EvaluationService {\n private engine: LLMEvaluationEngine;\n\n constructor() {\n this.engine = new LLMEvaluationEngine();\n }\n\n /\n Evaluates a test case response\n * @param testCase - The test case to evaluate\n * @param onResult - Callback to handle the evaluation result\n */\n async evaluateTestCase(\n testCase: TestCase,\n onResult: (result: EvaluationResult) => void,\n ): Promise<void> {\n if ~~(!testCase.output)~~ ~~{\n console.warn('⚠️~~ No ~~output~~ to ~~evaluate~~ ~~for~~ ~~test case:', testCase.id)~~;\n ~~return;\n }~~\n\n const ~~fields:~~ ~~FieldEvaluationInput[~~] = (testCase.expectedOutcome \|\| []).~~flatMap~~(~~\n (field, index~~) => {\n if (field.type === 'textarea' && field.outcomeMode === 'dynamic') {\n ~~return []~~;\n }\n\n ~~return~~ [\n {\n index,\n label: field.label,\n type: field.type,\n expectedValue: ~~getFieldExpectedValue(field)~~,\n evaluationParameters: ~~normalizeEvaluationParametersForField~~(\n field.~~type~~,\n field.evaluationParameters,\n ),\n },\n ];\n },\n );\n\n const evaluationRequest: EvaluationRequestV2 = {\n testCaseId: testCase.id,\n question: testCase.question,\n ~~actualResponse: testCase.output,\n~~ fields,\n };\n\n await this.engine.evaluateResponse(\n evaluationRequest~~,\n~~ (result: EvaluationResult) => {\n ~~console.log('📊~~ ~~Evaluation~~ result ~~received:'~~, ~~result~~);\n onResult(result);\n },\n );\n }\n}\n\nfunction getFieldExpectedValue(field: ExpectedOutcomeField): string {\n if (field.type === 'chips-input') {\n return field.value.join(', ');\n }\n return field.value;\n}\n"]}
1	+ {"version":3,"file":"evaluation-service.js","sourceRoot":"","sources":["../../../src/lib/evaluation/evaluation-service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAY1D,OAAO,EAAE,qCAAqC,EAAE,MAAM,6BAA6B,CAAC;AACpF,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAE7D;;GAEG;AACH,MAAM,OAAO,iBAAiB;IACpB,MAAM,CAAsB;IAEpC;QACE,IAAI,CAAC,MAAM,GAAG,IAAI,mBAAmB,EAAE,CAAC;IAC1C,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB,CACpB,QAAkB,EAClB,QAA4C,EAC5C,UAAuC;QAEvC,MAAM,MAAM,GAA2B,EAAE,CAAC;QAC1C,MAAM,YAAY,GAA4B,EAAE,CAAC;QAEjD,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;YACxE,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;gBACjE,SAAS;YACX,CAAC;YAED,MAAM,oBAAoB,GAAG,qCAAqC,CAChE,KAAK,CAAC,IAAI,EACV,KAAK,CAAC,oBAAoB,CAC3B,CAAC;YACF,MAAM,aAAa,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;YACnD,MAAM,mBAAmB,GAAG,MAAM,kBAAkB,CAClD,KAAK,EACL,QAAQ,CAAC,MAAM,EACf,UAAU,CACX,CAAC;YAEF,IAAI,mBAAmB,CAAC,OAAO,EAAE,CAAC;gBAChC,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK;oBACL,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,aAAa;oBACb,cAAc,EAAE,mBAAmB,CAAC,KAAK;oBACzC,oBAAoB;iBACrB,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,YAAY,CAAC,IAAI,CAAC;oBAChB,KAAK;oBACL,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,aAAa;oBACb,MAAM,EAAE,KAAK;oBACb,cAAc,EAAE,EAAE;oBAClB,oBAAoB;oBACpB,wBAAwB,EAAE;wBACxB,KAAK,EAAE,CAAC;wBACR,YAAY,EAAE,oBAAoB,CAAC,QAAQ;qBAC5C;oBACD,KAAK,EACH,OAAO,IAAI,mBAAmB;wBAC5B,CAAC,CAAC,mBAAmB,CAAC,KAAK;wBAC3B,CAAC,CAAC,iCAAiC;iBACxC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9B,OAAO,CAAC,IAAI,CAAC,uCAAuC,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACnE,OAAO;YACT,CAAC;YAED,QAAQ,CAAC;gBACP,UAAU,EAAE,QAAQ,CAAC,EAAE;gBACvB,MAAM,EAAE,KAAK;gBACb,cAAc,EAAE,EAAE;gBAClB,YAAY,EAAE,YAAY;gBAC1B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,MAAM,iBAAiB,GAAwB;YAC7C,UAAU,EAAE,QAAQ,CAAC,EAAE;YACvB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,MAAM;SACP,CAAC;QAEF,MAAM,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,CAAC,MAAwB,EAAE,EAAE;YACjF,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,CAAC,EAAE,GAAG,YAAY,CAAC,CAAC,IAAI,CAC5E,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAC5B,CAAC;YACF,QAAQ,CAAC;gBACP,GAAG,MAAM;gBACT,MAAM,EAAE,eAAe,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;gBACpE,YAAY,EAAE,eAAe;aAC9B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AAED,SAAS,qBAAqB,CAAC,KAA2B;IACxD,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;QACjC,OAAO,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,KAAK,CAAC,KAAK,CAAC;AACrB,CAAC","sourcesContent":["import { LLMEvaluationEngine } from './evaluation-engine';\nimport {\n EvaluationResult,\n FieldEvaluationInput,\n EvaluationRequestV2,\n FieldEvaluationResult,\n} from './types';\nimport {\n TestCase,\n ExpectedOutcomeField,\n EvaluationSourceExtractors,\n} from '../../types/llm-test-runner';\nimport { normalizeEvaluationParametersForField } from './field-evaluation-approach';\nimport { resolveActualValue } from './actual-value-resolver';\n\n/*\n Service for evaluating test case responses\n /\nexport class EvaluationService {\n private engine: LLMEvaluationEngine;\n\n constructor() {\n this.engine = new LLMEvaluationEngine();\n }\n\n /\n Evaluates a test case response\n * @param testCase - The test case to evaluate\n * @param onResult - Callback to handle the evaluation result\n */\n async evaluateTestCase(\n testCase: TestCase,\n onResult: (result: EvaluationResult) => void,\n extractors?: EvaluationSourceExtractors,\n ): Promise<void> {\n const fields: FieldEvaluationInput[] = [];\n const failedFields: FieldEvaluationResult[] = [];\n\n for (const [index, field] of (testCase.expectedOutcome \|\| []).entries()) {\n if (field.type === 'textarea' && field.outcomeMode === 'dynamic') {\n continue;\n }\n\n const evaluationParameters = normalizeEvaluationParametersForField(\n field.type,\n field.evaluationParameters,\n );\n const expectedValue = getFieldExpectedValue(field);\n const resolvedActualValue = await resolveActualValue(\n field,\n testCase.output,\n extractors,\n );\n\n if (resolvedActualValue.success) {\n fields.push({\n index,\n label: field.label,\n type: field.type,\n expectedValue,\n actualResponse: resolvedActualValue.value,\n evaluationParameters,\n });\n } else {\n failedFields.push({\n index,\n label: field.label,\n type: field.type,\n expectedValue,\n passed: false,\n keywordMatches: [],\n evaluationParameters,\n evaluationApproachResult: {\n score: 0,\n approachUsed: evaluationParameters.approach,\n },\n error:\n 'error' in resolvedActualValue\n ? resolvedActualValue.error\n : 'Failed to resolve actual value.',\n });\n }\n }\n\n if (fields.length === 0) {\n if (failedFields.length === 0) {\n console.warn('⚠️ No evaluable fields for test case:', testCase.id);\n return;\n }\n\n onResult({\n testCaseId: testCase.id,\n passed: false,\n keywordMatches: [],\n fieldResults: failedFields,\n timestamp: new Date().toISOString(),\n });\n return;\n }\n\n const evaluationRequest: EvaluationRequestV2 = {\n testCaseId: testCase.id,\n question: testCase.question,\n fields,\n };\n\n await this.engine.evaluateResponse(evaluationRequest, (result: EvaluationResult) => {\n const combinedResults = [...(result.fieldResults \|\| []), ...failedFields].sort(\n (a, b) => a.index - b.index,\n );\n onResult({\n ...result,\n passed: combinedResults.every(field => field.passed && !field.error),\n fieldResults: combinedResults,\n });\n });\n }\n}\n\nfunction getFieldExpectedValue(field: ExpectedOutcomeField): string {\n if (field.type === 'chips-input') {\n return field.value.join(', ');\n }\n return field.value;\n}\n"]}

package/dist/collection/lib/evaluation/types.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/lib/evaluation/types.ts"],"names":[],"mappings":"","sourcesContent":["import {\n EvaluationParameters,\n EvaluationApproachResult,\n} from '../../types/evaluation';\nimport type { ExpectedOutcomeFieldType } from '../../types/llm-test-runner';\n\nexport interface EvaluationRequest {\n testCaseId: string;\n question: string;\n expectedOutcome: string;\n actualResponse: string;\n evaluationParameters: EvaluationParameters;\n}\n\nexport interface FieldEvaluationInput {\n index: number;\n label: string;\n type: ExpectedOutcomeFieldType;\n expectedValue: string;\n evaluationParameters: EvaluationParameters;\n}\n\nexport interface EvaluationRequestV2 {\n testCaseId: string;\n question: string;\n ~~actualResponse: string;\n~~ fields: FieldEvaluationInput[];\n}\n\nexport interface EvaluationResult {\n testCaseId: string;\n passed: boolean;\n keywordMatches: KeywordMatch[];\n fieldResults?: FieldEvaluationResult[];\n timestamp?: string;\n evaluationParameters?: EvaluationParameters;\n evaluationApproachResult?: EvaluationApproachResult;\n}\n\nexport interface FieldEvaluationResult {\n index: number;\n label: string;\n type: ExpectedOutcomeFieldType;\n expectedValue: string;\n passed: boolean;\n keywordMatches: KeywordMatch[];\n evaluationParameters: EvaluationParameters;\n evaluationApproachResult: EvaluationApproachResult;\n error?: string;\n}\n\nexport interface KeywordMatch {\n keyword: string;\n found: boolean;\n evaluationApproachResult: EvaluationApproachResult;\n}\n\nexport type EvaluationCallback = (result: EvaluationResult) => void;\n\nexport interface RougeKeywordDetails {\n rouge1: number;\n rougeL: number;\n scoreUsed: string;\n approach: string;\n}\n\nexport interface Rouge1OverallDetails {\n keywordsPassed: number;\n totalKeywords: number;\n passRate: string;\n thresholdUsed: number;\n approach: string;\n}\n"]}
1	+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/lib/evaluation/types.ts"],"names":[],"mappings":"","sourcesContent":["import {\n EvaluationParameters,\n EvaluationApproachResult,\n} from '../../types/evaluation';\nimport type { ExpectedOutcomeFieldType } from '../../types/llm-test-runner';\n\nexport interface EvaluationRequest {\n testCaseId: string;\n question: string;\n expectedOutcome: string;\n actualResponse: string;\n evaluationParameters: EvaluationParameters;\n}\n\nexport interface FieldEvaluationInput {\n index: number;\n label: string;\n type: ExpectedOutcomeFieldType;\n expectedValue: string;\n actualResponse: string;\n evaluationParameters: EvaluationParameters;\n}\n\nexport interface EvaluationRequestV2 {\n testCaseId: string;\n question: string;\n fields: FieldEvaluationInput[];\n}\n\nexport interface EvaluationResult {\n testCaseId: string;\n passed: boolean;\n keywordMatches: KeywordMatch[];\n fieldResults?: FieldEvaluationResult[];\n timestamp?: string;\n evaluationParameters?: EvaluationParameters;\n evaluationApproachResult?: EvaluationApproachResult;\n}\n\nexport interface FieldEvaluationResult {\n index: number;\n label: string;\n type: ExpectedOutcomeFieldType;\n expectedValue: string;\n passed: boolean;\n keywordMatches: KeywordMatch[];\n evaluationParameters: EvaluationParameters;\n evaluationApproachResult: EvaluationApproachResult;\n error?: string;\n}\n\nexport interface KeywordMatch {\n keyword: string;\n found: boolean;\n evaluationApproachResult: EvaluationApproachResult;\n}\n\nexport type EvaluationCallback = (result: EvaluationResult) => void;\n\nexport interface RougeKeywordDetails {\n rouge1: number;\n rougeL: number;\n scoreUsed: string;\n approach: string;\n}\n\nexport interface Rouge1OverallDetails {\n keywordsPassed: number;\n totalKeywords: number;\n passRate: string;\n thresholdUsed: number;\n approach: string;\n}\n"]}

package/dist/collection/lib/form/components/app-chips.js CHANGED Viewed

@@ -29,7 +29,7 @@ export class AppChips {
             name: c.name,
             autocomplete: c.autocomplete,
         };
-        return (h("div", { key: '4f081007c0b11fb20bd74644eb063a6dd5a45d98', class: "app-chips" }, c.label && (h("label", { key: '3934ec12898c947b163a15494330a9326fe3d6f1', class: "app-chips__label", htmlFor: c.name }, c.label)), h("div", { key: 'cba194df0033a5951f568875e4686a76c9c28438', class: "app-chips__container" }, this.value.map((chip) => (h("span", { class: "app-chips__chip", key: chip }, c.type === 'url' ? (h("a", { href: chip, target: "_blank", rel: "noopener noreferrer", class: "app-chips__link" }, chip)) : (chip), h("button", { class: "app-chips__remove", type: "button", onClick: () => this.emitRemoveChip(chip) }, "\u00D7")))), h("input", { key: '11c9e7f326c37cb3a413e9124926ca88eefb1911', class: "app-chips__input", type: c.type || 'text', ...allowedAttrs, onKeyDown: (e) => {
+        return (h("div", { key: 'fb7d4d5444e9c9ac33c56aec88e3e10ed103c8be', class: "app-chips" }, c.label && (h("label", { key: '2d0041b3a137fecddef2273eac3792b5e8de27ab', class: "app-chips__label", htmlFor: c.name }, c.label)), h("div", { key: 'f73b1105e567b233626073e05b9da712689e7b12', class: "app-chips__container" }, this.value.map((chip) => (h("span", { class: "app-chips__chip", key: chip }, c.type === 'url' ? (h("a", { href: chip, target: "_blank", rel: "noopener noreferrer", class: "app-chips__link" }, chip)) : (chip), h("button", { class: "app-chips__remove", type: "button", onClick: () => this.emitRemoveChip(chip) }, "\u00D7")))), h("input", { key: '7676ff95531b34d10cbf9402e72a723e7e123e0e', class: "app-chips__input", type: c.type || 'text', ...allowedAttrs, onKeyDown: (e) => {
                 if (e.key === 'Enter') {
                     const input = e.target;
                     const val = input.value.trim();

package/dist/collection/lib/form/components/app-select.js CHANGED Viewed

@@ -14,7 +14,7 @@ export class AppSelect {
             placeholder: c.placeholder,
             autocomplete: c.autocomplete,
         };
-        return (h("div", { key: '4d9d250026b51799b073ae828c7d15563ec27c22', class: "app-select" }, c.label && (h("label", { key: '53e101427e78063bd6ed671a31919bbe373c5563', class: "app-select__label", htmlFor: c.name }, c.label)), h("div", { key: '28b18245c261a3766948ed7d151c356b8faeb4cf' }, h("select", { key: 'eed9bcb9fc3e00ae62c681a37e967815d350d5fb', ...allowedAttrs, class: "app-select__select", onInput: (e) => {
+        return (h("div", { key: '968f0fffe1eff976ac7e00f02db6fb84aa529de4', class: "app-select" }, c.label && (h("label", { key: 'dac02d2335754ff5d6ce6ba1df5777f8b019cfae', class: "app-select__label", htmlFor: c.name }, c.label)), h("div", { key: 'c92698199479bbdde1cfd559d69fc97e54d2862a' }, h("select", { key: '6e9b27b034d057921f52fbd12653c5ef1b6af1bc', ...allowedAttrs, class: "app-select__select", onInput: (e) => {
                 const raw = e.target.value;
                 const matched = c.optionList.find(opt => String(opt) === raw);
                 this.valueChange.emit({

package/dist/collection/lib/form/components/app-textarea.js CHANGED Viewed

@@ -21,10 +21,10 @@ export class AppTextarea {
             name: c.name,
             autocomplete: c.autocomplete,
         };
-        return (h("div", { key: '29bbcd954df6d18e968f6b2670224fda6f03a9f2', class: {
+        return (h("div", { key: 'f0749b1f2badf8434272da9fb37b354b42ba988b', class: {
                 'textarea-wrapper': true,
                 'textarea-wrapper--read-only': !!c.readOnly,
-            } }, c.label && (h("label", { key: 'd95d3cbf01ce66c5bd074ede2eb1f87a0c98f3e4', class: "textarea-label", htmlFor: c.name }, c.label)), h("textarea", { key: 'faeb28c35a543213d9c5847e93779f957e367ff0', ...allowedAttrs, class: "textarea-element", value: this.value, onInput: this.handleChange }), c.helpText && h("p", { key: '558abfe03f7107352a6d40aea78a35b304bc11a6', class: "help-text" }, c.helpText)));
+            } }, c.label && (h("label", { key: '3448c838bcf9e962df005eae8fc313d216497c35', class: "textarea-label", htmlFor: c.name }, c.label)), h("textarea", { key: 'b4ee67a24fa71b0fa042625f943b0e226a6d14b7', ...allowedAttrs, class: "textarea-element", value: this.value, onInput: this.handleChange }), c.helpText && h("p", { key: 'fb6263c32e6cc5e36dbc77344c31487d63d51a1c', class: "help-text" }, c.helpText)));
     }
     static get is() { return "app-textarea"; }
     static get encapsulation() { return "shadow"; }

package/dist/collection/lib/import-export/test-suite-exporter.js CHANGED Viewed

@@ -8,6 +8,10 @@ export function formatTestSuiteAsJson(testCases) {
         id: testCase.id,
         question: testCase.question,
         expectedOutcome: testCase.expectedOutcome,
+        chatHistory: {
+            enabled: testCase.chatHistory.enabled,
+            value: testCase.chatHistory.value,
+        },
     }));
     return JSON.stringify(exportData, null, 2);
 }

package/dist/collection/lib/import-export/test-suite-exporter.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"test-suite-exporter.js","sourceRoot":"","sources":["../../../src/lib/import-export/test-suite-exporter.ts"],"names":[],"mappings":"~~AAQA~~;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,SAAqB;IACzD,MAAM,UAAU,GAA0B,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACnE,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,eAAe,EAAE,QAAQ,CAAC,eAAe;~~KAC1C~~,CAAC,CAAC,CAAC;IAEJ,OAAO,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AAC7C,CAAC","sourcesContent":["import { ExpectedOutcomeField, TestCase } from '../../types/llm-test-runner';\n\nexport interface TestSuiteExportData {\n id: string;\n question: string;\n expectedOutcome: ExpectedOutcomeField[];\n}\n\n/*\n Formats test cases as a JSON string suitable for saving as a test suite\n * @param testCases - Array of test cases to format\n * @returns JSON string representation of the test suite\n */\nexport function formatTestSuiteAsJson(testCases: TestCase[]): string {\n const exportData: TestSuiteExportData[] = testCases.map(testCase => ({\n id: testCase.id,\n question: testCase.question,\n expectedOutcome: testCase.expectedOutcome,\n }));\n\n return JSON.stringify(exportData, null, 2);\n}\n"]}
1	+ {"version":3,"file":"test-suite-exporter.js","sourceRoot":"","sources":["../../../src/lib/import-export/test-suite-exporter.ts"],"names":[],"mappings":"AAYA;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,SAAqB;IACzD,MAAM,UAAU,GAA0B,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACnE,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,eAAe,EAAE,QAAQ,CAAC,eAAe;QACzC,WAAW,EAAE;YACX,OAAO,EAAE,QAAQ,CAAC,WAAW,CAAC,OAAO;YACrC,KAAK,EAAE,QAAQ,CAAC,WAAW,CAAC,KAAK;SAClC;KACF,CAAC,CAAC,CAAC;IAEJ,OAAO,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AAC7C,CAAC","sourcesContent":["import { ExpectedOutcomeField, TestCase } from '../../types/llm-test-runner';\n\nexport interface TestSuiteExportData {\n id: string;\n question: string;\n expectedOutcome: ExpectedOutcomeField[];\n chatHistory: {\n enabled: boolean;\n value: string;\n };\n}\n\n/*\n Formats test cases as a JSON string suitable for saving as a test suite\n * @param testCases - Array of test cases to format\n * @returns JSON string representation of the test suite\n */\nexport function formatTestSuiteAsJson(testCases: TestCase[]): string {\n const exportData: TestSuiteExportData[] = testCases.map(testCase => ({\n id: testCase.id,\n question: testCase.question,\n expectedOutcome: testCase.expectedOutcome,\n chatHistory: {\n enabled: testCase.chatHistory.enabled,\n value: testCase.chatHistory.value,\n },\n }));\n\n return JSON.stringify(exportData, null, 2);\n}\n"]}

package/dist/collection/lib/import-export/test-suite-importer.js CHANGED Viewed

@@ -1,14 +1,20 @@
 import { createTestCaseFromInput } from "../test-cases/test-case-factory";
 import { validateTestCaseInputArray } from "../../schemas/test-case";
+import { validateExpectedOutcomeArrayWithExtractors } from "../../schemas/expected-outcome";
 /**
  * Validates and imports test cases from JSON content
  * @param jsonContent - The JSON string to parse and validate
  * @returns Validation result with test cases or error message
  */
-export function importTestSuite(jsonContent) {
+export function importTestSuite(jsonContent, allowedExtractorIds = []) {
     try {
         const parsed = JSON.parse(jsonContent);
         validateTestCaseInputArray(parsed);
+        if (allowedExtractorIds.length > 0) {
+            parsed.forEach((testCase) => {
+                validateExpectedOutcomeArrayWithExtractors(testCase.expectedOutcome, allowedExtractorIds);
+            });
+        }
         const testCases = parsed.map((item, index) => {
             try {
                 return createTestCaseFromInput(item);

package/dist/collection/lib/import-export/test-suite-importer.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"test-suite-importer.js","sourceRoot":"","sources":["../../../src/lib/import-export/test-suite-importer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAC1E,OAAO,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;~~AAQrE~~;;;;GAIG;AACH,MAAM,UAAU,eAAe,~~CAAC~~,WAAmB;~~IACjD~~,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACvC,0BAA0B,CAAC,MAAM,CAAC,CAAC;~~QAEnC~~,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YAC3C,IAAI,CAAC;gBACH,OAAO,uBAAuB,CAAC,IAAI,CAAC,CAAC;YACvC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;gBACrE,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,KAAK,OAAO,EAAE,CAAC,CAAC;YACrE,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,SAAS;SACV,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EACH,GAAG,YAAY,KAAK;gBAClB,CAAC,CAAC,GAAG,CAAC,OAAO;gBACb,CAAC,CAAC,gEAAgE;SACvE,CAAC;IACJ,CAAC;AACH,CAAC","sourcesContent":["import type { TestCase } from '../../types/llm-test-runner';\nimport { createTestCaseFromInput } from '../test-cases/test-case-factory';\nimport { validateTestCaseInputArray } from '../../schemas/test-case';\n\nexport interface ImportValidationResult {\n success: boolean;\n testCases?: TestCase[];\n error?: string;\n}\n\n/*\n Validates and imports test cases from JSON content\n * @param jsonContent - The JSON string to parse and validate\n * @returns Validation result with test cases or error message\n */\nexport function importTestSuite(jsonContent: string): ImportValidationResult {\n try {\n const parsed = JSON.parse(jsonContent);\n validateTestCaseInputArray(parsed);\n\n const testCases = parsed.map((item, index) => {\n try {\n return createTestCaseFromInput(item);\n } catch (err) {\n const message = err instanceof Error ? err.message : 'Unknown error';\n throw new Error(`Invalid test case at index ${index}: ${message}`);\n }\n });\n\n return {\n success: true,\n testCases,\n };\n } catch (err) {\n return {\n success: false,\n error:\n err instanceof Error\n ? err.message\n : 'Error processing file. Please ensure it is a valid JSON array.',\n };\n }\n}\n\n"]}
1	+ {"version":3,"file":"test-suite-importer.js","sourceRoot":"","sources":["../../../src/lib/import-export/test-suite-importer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAC1E,OAAO,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;AACrE,OAAO,EAAE,0CAA0C,EAAE,MAAM,gCAAgC,CAAC;AAQ5F;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAC7B,WAAmB,EACnB,sBAAgC,EAAE;IAElC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACvC,0BAA0B,CAAC,MAAM,CAAC,CAAC;QACnC,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;gBAC1B,0CAA0C,CACxC,QAAQ,CAAC,eAAe,EACxB,mBAAmB,CACpB,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YAC3C,IAAI,CAAC;gBACH,OAAO,uBAAuB,CAAC,IAAI,CAAC,CAAC;YACvC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;gBACrE,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,KAAK,OAAO,EAAE,CAAC,CAAC;YACrE,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,SAAS;SACV,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EACH,GAAG,YAAY,KAAK;gBAClB,CAAC,CAAC,GAAG,CAAC,OAAO;gBACb,CAAC,CAAC,gEAAgE;SACvE,CAAC;IACJ,CAAC;AACH,CAAC","sourcesContent":["import type { TestCase } from '../../types/llm-test-runner';\nimport { createTestCaseFromInput } from '../test-cases/test-case-factory';\nimport { validateTestCaseInputArray } from '../../schemas/test-case';\nimport { validateExpectedOutcomeArrayWithExtractors } from '../../schemas/expected-outcome';\n\nexport interface ImportValidationResult {\n success: boolean;\n testCases?: TestCase[];\n error?: string;\n}\n\n/*\n Validates and imports test cases from JSON content\n * @param jsonContent - The JSON string to parse and validate\n * @returns Validation result with test cases or error message\n */\nexport function importTestSuite(\n jsonContent: string,\n allowedExtractorIds: string[] = [],\n): ImportValidationResult {\n try {\n const parsed = JSON.parse(jsonContent);\n validateTestCaseInputArray(parsed);\n if (allowedExtractorIds.length > 0) {\n parsed.forEach((testCase) => {\n validateExpectedOutcomeArrayWithExtractors(\n testCase.expectedOutcome,\n allowedExtractorIds,\n );\n });\n }\n\n const testCases = parsed.map((item, index) => {\n try {\n return createTestCaseFromInput(item);\n } catch (err) {\n const message = err instanceof Error ? err.message : 'Unknown error';\n throw new Error(`Invalid test case at index ${index}: ${message}`);\n }\n });\n\n return {\n success: true,\n testCases,\n };\n } catch (err) {\n return {\n success: false,\n error:\n err instanceof Error\n ? err.message\n : 'Error processing file. Please ensure it is a valid JSON array.',\n };\n }\n}\n\n"]}

package/dist/collection/lib/test-cases/test-case-factory.js CHANGED Viewed

@@ -11,6 +11,7 @@ export const DEFAULT_EXPECTED_OUTCOME_SCHEMA = [
 function normalizeExpectedOutcomeField(field) {
     return {
         ...field,
+        evaluationSource: field.evaluationSource || { type: 'text' },
         evaluationParameters: normalizeEvaluationParametersForField(field.type, field.evaluationParameters),
     };
 }
@@ -23,6 +24,7 @@ export function createTestCase(expectedOutcomeSchema = DEFAULT_EXPECTED_OUTCOME_
         id: uuidv4(),
         question: '',
         expectedOutcome: createExpectedOutcomeFromSchema(expectedOutcomeSchema),
+        chatHistory: { enabled: false, value: '' },
         isRunning: false,
     };
 }
@@ -33,6 +35,7 @@ function createExpectedOutcomeFieldFromSchema(schemaField) {
                 type: 'text',
                 label: schemaField.label,
                 placeholder: schemaField.placeholder,
+                evaluationSource: schemaField.evaluationSource || { type: 'text' },
                 value: '',
                 evaluationParameters: normalizeEvaluationParametersForField(schemaField.type, schemaField.evaluationParameters),
             };
@@ -41,6 +44,7 @@ function createExpectedOutcomeFieldFromSchema(schemaField) {
                 type: 'textarea',
                 label: schemaField.label,
                 placeholder: schemaField.placeholder,
+                evaluationSource: schemaField.evaluationSource || { type: 'text' },
                 rows: schemaField.rows,
                 value: '',
                 evaluationParameters: normalizeEvaluationParametersForField(schemaField.type, schemaField.evaluationParameters),
@@ -50,6 +54,7 @@ function createExpectedOutcomeFieldFromSchema(schemaField) {
                 type: 'chips-input',
                 label: schemaField.label,
                 placeholder: schemaField.placeholder,
+                evaluationSource: schemaField.evaluationSource || { type: 'text' },
                 value: [],
                 evaluationParameters: normalizeEvaluationParametersForField(schemaField.type, schemaField.evaluationParameters),
             };
@@ -58,6 +63,7 @@ function createExpectedOutcomeFieldFromSchema(schemaField) {
                 type: 'select',
                 label: schemaField.label,
                 placeholder: schemaField.placeholder,
+                evaluationSource: schemaField.evaluationSource || { type: 'text' },
                 value: schemaField.options[0],
                 options: schemaField.options,
                 evaluationParameters: normalizeEvaluationParametersForField(schemaField.type, schemaField.evaluationParameters),
@@ -82,6 +88,7 @@ export function createExpectedOutcomeFromSchema(expectedOutcomeSchema) {
 export function createTestCaseFromInput(data) {
     return {
         ...data,
+        chatHistory: data.chatHistory ?? { enabled: false, value: '' },
         expectedOutcome: data.expectedOutcome.map(normalizeExpectedOutcomeField),
     };
 }

package/dist/collection/lib/test-cases/test-case-factory.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"test-case-factory.js","sourceRoot":"","sources":["../../../src/lib/test-cases/test-case-factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AASpC,OAAO,EAAE,qCAAqC,EAAE,MAAM,yCAAyC,CAAC;AAEhG,MAAM,CAAC,MAAM,+BAA+B,GAA0B;IACpE;QACE,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,kBAAkB;QACzB,WAAW,EAAE,2BAA2B;QACxC,IAAI,EAAE,CAAC;KACR;CACF,CAAC;AAEF,SAAS,6BAA6B,CACpC,KAA2B;IAE3B,OAAO;QACL,GAAG,KAAK;QACR,oBAAoB,EAAE,qCAAqC,CACzD,KAAK,CAAC,IAAI,EACV,KAAK,CAAC,oBAAoB,CAC3B;KACF,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAC5B,wBAA+C,+BAA+B;IAE9E,OAAO;QACL,EAAE,EAAE,MAAM,EAAE;QACZ,QAAQ,EAAE,EAAE;QACZ,eAAe,EAAE,+BAA+B,CAAC,qBAAqB,CAAC;QACvE,SAAS,EAAE,KAAK;KACjB,CAAC;AACJ,CAAC;AAED,SAAS,oCAAoC,CAC3C,WAAuC;IAEvC,QAAQ,WAAW,CAAC,IAAI,EAAE,CAAC;QACzB,KAAK,MAAM;YACT,OAAO;gBACL,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,WAAW,EAAE,WAAW,CAAC,WAAW;gBACpC,KAAK,EAAE,EAAE;gBACT,oBAAoB,EAAE,qCAAqC,CACzD,WAAW,CAAC,IAAI,EAChB,WAAW,CAAC,oBAAoB,CACjC;aACF,CAAC;QAEJ,KAAK,UAAU;YACb,OAAO;gBACL,IAAI,EAAE,UAAU;gBAChB,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,WAAW,EAAE,WAAW,CAAC,WAAW;gBACpC,IAAI,EAAE,WAAW,CAAC,IAAI;gBACtB,KAAK,EAAE,EAAE;gBACT,oBAAoB,EAAE,qCAAqC,CACzD,WAAW,CAAC,IAAI,EAChB,WAAW,CAAC,oBAAoB,CACjC;aACF,CAAC;QAEJ,KAAK,aAAa;YAChB,OAAO;gBACL,IAAI,EAAE,aAAa;gBACnB,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,WAAW,EAAE,WAAW,CAAC,WAAW;gBACpC,KAAK,EAAE,EAAE;gBACT,oBAAoB,EAAE,qCAAqC,CACzD,WAAW,CAAC,IAAI,EAChB,WAAW,CAAC,oBAAoB,CACjC;aACF,CAAC;QAEJ,KAAK,QAAQ;YACX,OAAO;gBACL,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,WAAW,EAAE,WAAW,CAAC,WAAW;gBACpC,KAAK,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC7B,OAAO,EAAE,WAAW,CAAC,OAAO;gBAC5B,oBAAoB,EAAE,qCAAqC,CACzD,WAAW,CAAC,IAAI,EAChB,WAAW,CAAC,oBAAoB,CAC6B;aAChE,CAAC;QAEJ,OAAO,CAAC,CAAC,CAAC;YACR,MAAM,gBAAgB,GAAU,WAAW,CAAC;YAC5C,OAAO,gBAAgB,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,+BAA+B,CAC7C,qBAA4C;IAE5C,OAAO,qBAAqB,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;AACzE,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,uBAAuB,CAAC,IAAmB;IACzD,OAAO;QACL,GAAG,IAAI;QACP,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,6BAA6B,CAAC;KACzE,CAAC;AACJ,CAAC","sourcesContent":["import { v4 as uuidv4 } from 'uuid';\nimport {\n ExpectedOutcomeField,\n ExpectedOutcomeSchema,\n ExpectedOutcomeSchemaField,\n TestCase,\n TestCaseInput,\n} from '../../types/llm-test-runner';\nimport { EvaluationApproach } from '../evaluation/constants';\nimport { normalizeEvaluationParametersForField } from '../evaluation/field-evaluation-approach';\n\nexport const DEFAULT_EXPECTED_OUTCOME_SCHEMA: ExpectedOutcomeSchema = [\n {\n type: 'textarea',\n label: 'Expected Outcome',\n placeholder: 'Enter expected outcome...',\n rows: 2,\n },\n];\n\nfunction normalizeExpectedOutcomeField(\n field: ExpectedOutcomeField,\n): ExpectedOutcomeField {\n return {\n ...field,\n evaluationParameters: normalizeEvaluationParametersForField(\n field.type,\n field.evaluationParameters,\n ),\n };\n}\n\n/*\n Creates a new test case with default values\n * @returns A new TestCase object with a unique ID\n /\nexport function createTestCase(\n expectedOutcomeSchema: ExpectedOutcomeSchema = DEFAULT_EXPECTED_OUTCOME_SCHEMA,\n): TestCase {\n return {\n id: uuidv4(),\n question: '',\n expectedOutcome: createExpectedOutcomeFromSchema(expectedOutcomeSchema),\n isRunning: false,\n };\n}\n\nfunction createExpectedOutcomeFieldFromSchema(\n schemaField: ExpectedOutcomeSchemaField,\n): ExpectedOutcomeField {\n switch (schemaField.type) {\n case 'text':\n return {\n type: 'text',\n label: schemaField.label,\n placeholder: schemaField.placeholder,\n value: '',\n evaluationParameters: normalizeEvaluationParametersForField(\n schemaField.type,\n schemaField.evaluationParameters,\n ),\n };\n\n case 'textarea':\n return {\n type: 'textarea',\n label: schemaField.label,\n placeholder: schemaField.placeholder,\n rows: schemaField.rows,\n value: '',\n evaluationParameters: normalizeEvaluationParametersForField(\n schemaField.type,\n schemaField.evaluationParameters,\n ),\n };\n\n case 'chips-input':\n return {\n type: 'chips-input',\n label: schemaField.label,\n placeholder: schemaField.placeholder,\n value: [],\n evaluationParameters: normalizeEvaluationParametersForField(\n schemaField.type,\n schemaField.evaluationParameters,\n ),\n };\n\n case 'select':\n return {\n type: 'select',\n label: schemaField.label,\n placeholder: schemaField.placeholder,\n value: schemaField.options[0],\n options: schemaField.options,\n evaluationParameters: normalizeEvaluationParametersForField(\n schemaField.type,\n schemaField.evaluationParameters,\n ) as { approach: EvaluationApproach.EXACT; threshold?: number },\n };\n\n default: {\n const _exhaustiveCheck: never = schemaField;\n return _exhaustiveCheck;\n }\n }\n}\n\nexport function createExpectedOutcomeFromSchema(\n expectedOutcomeSchema: ExpectedOutcomeSchema,\n): ExpectedOutcomeField[] {\n return expectedOutcomeSchema.map(createExpectedOutcomeFieldFromSchema);\n}\n\n/\n Creates a runtime test case from validated input data.\n * The input is expected to already satisfy `TestCaseInput`,\n * and this function only performs normalization/defaulting.\n \n @param data - Validated test case input\n * @returns A normalized TestCase object with runtime defaults applied\n */\nexport function createTestCaseFromInput(data: TestCaseInput): TestCase {\n return {\n ...data,\n expectedOutcome: data.expectedOutcome.map(normalizeExpectedOutcomeField),\n };\n}\n"]}
1	+ {"version":3,"file":"test-case-factory.js","sourceRoot":"","sources":["../../../src/lib/test-cases/test-case-factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AASpC,OAAO,EAAE,qCAAqC,EAAE,MAAM,yCAAyC,CAAC;AAEhG,MAAM,CAAC,MAAM,+BAA+B,GAA0B;IACpE;QACE,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,kBAAkB;QACzB,WAAW,EAAE,2BAA2B;QACxC,IAAI,EAAE,CAAC;KACR;CACF,CAAC;AAEF,SAAS,6BAA6B,CACpC,KAA2B;IAE3B,OAAO;QACL,GAAG,KAAK;QACR,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE;QAC5D,oBAAoB,EAAE,qCAAqC,CACzD,KAAK,CAAC,IAAI,EACV,KAAK,CAAC,oBAAoB,CAC3B;KACF,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAC5B,wBAA+C,+BAA+B;IAE9E,OAAO;QACL,EAAE,EAAE,MAAM,EAAE;QACZ,QAAQ,EAAE,EAAE;QACZ,eAAe,EAAE,+BAA+B,CAAC,qBAAqB,CAAC;QACvE,WAAW,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE;QAC1C,SAAS,EAAE,KAAK;KACjB,CAAC;AACJ,CAAC;AAED,SAAS,oCAAoC,CAC3C,WAAuC;IAEvC,QAAQ,WAAW,CAAC,IAAI,EAAE,CAAC;QACzB,KAAK,MAAM;YACT,OAAO;gBACL,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,WAAW,EAAE,WAAW,CAAC,WAAW;gBACpC,gBAAgB,EAAE,WAAW,CAAC,gBAAgB,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE;gBAClE,KAAK,EAAE,EAAE;gBACT,oBAAoB,EAAE,qCAAqC,CACzD,WAAW,CAAC,IAAI,EAChB,WAAW,CAAC,oBAAoB,CACjC;aACF,CAAC;QAEJ,KAAK,UAAU;YACb,OAAO;gBACL,IAAI,EAAE,UAAU;gBAChB,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,WAAW,EAAE,WAAW,CAAC,WAAW;gBACpC,gBAAgB,EAAE,WAAW,CAAC,gBAAgB,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE;gBAClE,IAAI,EAAE,WAAW,CAAC,IAAI;gBACtB,KAAK,EAAE,EAAE;gBACT,oBAAoB,EAAE,qCAAqC,CACzD,WAAW,CAAC,IAAI,EAChB,WAAW,CAAC,oBAAoB,CACjC;aACF,CAAC;QAEJ,KAAK,aAAa;YAChB,OAAO;gBACL,IAAI,EAAE,aAAa;gBACnB,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,WAAW,EAAE,WAAW,CAAC,WAAW;gBACpC,gBAAgB,EAAE,WAAW,CAAC,gBAAgB,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE;gBAClE,KAAK,EAAE,EAAE;gBACT,oBAAoB,EAAE,qCAAqC,CACzD,WAAW,CAAC,IAAI,EAChB,WAAW,CAAC,oBAAoB,CACjC;aACF,CAAC;QAEJ,KAAK,QAAQ;YACX,OAAO;gBACL,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,WAAW,EAAE,WAAW,CAAC,WAAW;gBACpC,gBAAgB,EAAE,WAAW,CAAC,gBAAgB,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE;gBAClE,KAAK,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC7B,OAAO,EAAE,WAAW,CAAC,OAAO;gBAC5B,oBAAoB,EAAE,qCAAqC,CACzD,WAAW,CAAC,IAAI,EAChB,WAAW,CAAC,oBAAoB,CAC6B;aAChE,CAAC;QAEJ,OAAO,CAAC,CAAC,CAAC;YACR,MAAM,gBAAgB,GAAU,WAAW,CAAC;YAC5C,OAAO,gBAAgB,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,+BAA+B,CAC7C,qBAA4C;IAE5C,OAAO,qBAAqB,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;AACzE,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,uBAAuB,CAAC,IAAmB;IACzD,OAAO;QACL,GAAG,IAAI;QACP,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE;QAC9D,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,6BAA6B,CAAC;KACzE,CAAC;AACJ,CAAC","sourcesContent":["import { v4 as uuidv4 } from 'uuid';\nimport {\n ExpectedOutcomeField,\n ExpectedOutcomeSchema,\n ExpectedOutcomeSchemaField,\n TestCase,\n TestCaseInput,\n} from '../../types/llm-test-runner';\nimport { EvaluationApproach } from '../evaluation/constants';\nimport { normalizeEvaluationParametersForField } from '../evaluation/field-evaluation-approach';\n\nexport const DEFAULT_EXPECTED_OUTCOME_SCHEMA: ExpectedOutcomeSchema = [\n {\n type: 'textarea',\n label: 'Expected Outcome',\n placeholder: 'Enter expected outcome...',\n rows: 2,\n },\n];\n\nfunction normalizeExpectedOutcomeField(\n field: ExpectedOutcomeField,\n): ExpectedOutcomeField {\n return {\n ...field,\n evaluationSource: field.evaluationSource \|\| { type: 'text' },\n evaluationParameters: normalizeEvaluationParametersForField(\n field.type,\n field.evaluationParameters,\n ),\n };\n}\n\n/*\n Creates a new test case with default values\n * @returns A new TestCase object with a unique ID\n /\nexport function createTestCase(\n expectedOutcomeSchema: ExpectedOutcomeSchema = DEFAULT_EXPECTED_OUTCOME_SCHEMA,\n): TestCase {\n return {\n id: uuidv4(),\n question: '',\n expectedOutcome: createExpectedOutcomeFromSchema(expectedOutcomeSchema),\n chatHistory: { enabled: false, value: '' },\n isRunning: false,\n };\n}\n\nfunction createExpectedOutcomeFieldFromSchema(\n schemaField: ExpectedOutcomeSchemaField,\n): ExpectedOutcomeField {\n switch (schemaField.type) {\n case 'text':\n return {\n type: 'text',\n label: schemaField.label,\n placeholder: schemaField.placeholder,\n evaluationSource: schemaField.evaluationSource \|\| { type: 'text' },\n value: '',\n evaluationParameters: normalizeEvaluationParametersForField(\n schemaField.type,\n schemaField.evaluationParameters,\n ),\n };\n\n case 'textarea':\n return {\n type: 'textarea',\n label: schemaField.label,\n placeholder: schemaField.placeholder,\n evaluationSource: schemaField.evaluationSource \|\| { type: 'text' },\n rows: schemaField.rows,\n value: '',\n evaluationParameters: normalizeEvaluationParametersForField(\n schemaField.type,\n schemaField.evaluationParameters,\n ),\n };\n\n case 'chips-input':\n return {\n type: 'chips-input',\n label: schemaField.label,\n placeholder: schemaField.placeholder,\n evaluationSource: schemaField.evaluationSource \|\| { type: 'text' },\n value: [],\n evaluationParameters: normalizeEvaluationParametersForField(\n schemaField.type,\n schemaField.evaluationParameters,\n ),\n };\n\n case 'select':\n return {\n type: 'select',\n label: schemaField.label,\n placeholder: schemaField.placeholder,\n evaluationSource: schemaField.evaluationSource \|\| { type: 'text' },\n value: schemaField.options[0],\n options: schemaField.options,\n evaluationParameters: normalizeEvaluationParametersForField(\n schemaField.type,\n schemaField.evaluationParameters,\n ) as { approach: EvaluationApproach.EXACT; threshold?: number },\n };\n\n default: {\n const _exhaustiveCheck: never = schemaField;\n return _exhaustiveCheck;\n }\n }\n}\n\nexport function createExpectedOutcomeFromSchema(\n expectedOutcomeSchema: ExpectedOutcomeSchema,\n): ExpectedOutcomeField[] {\n return expectedOutcomeSchema.map(createExpectedOutcomeFieldFromSchema);\n}\n\n/\n Creates a runtime test case from validated input data.\n * The input is expected to already satisfy `TestCaseInput`,\n * and this function only performs normalization/defaulting.\n \n @param data - Validated test case input\n * @returns A normalized TestCase object with runtime defaults applied\n */\nexport function createTestCaseFromInput(data: TestCaseInput): TestCase {\n return {\n ...data,\n chatHistory: data.chatHistory ?? { enabled: false, value: '' },\n expectedOutcome: data.expectedOutcome.map(normalizeExpectedOutcomeField),\n };\n}\n"]}

package/dist/collection/lib/test-cases/test-case-mutations.js CHANGED Viewed

@@ -1,4 +1,13 @@
 import { normalizeEvaluationParametersForField } from "../evaluation/field-evaluation-approach";
+function isChipsInputField(field) {
+    return field.type === 'chips-input';
+}
+function isTextareaField(field) {
+    return field.type === 'textarea';
+}
+function isDynamicTextareaField(field) {
+    return isTextareaField(field) && field.outcomeMode === 'dynamic';
+}
 export function applyExpectedOutcomeChange(testCase, change) {
     const { index } = change;
     const expectedOutcome = [...(testCase.expectedOutcome || [])];
@@ -6,73 +15,99 @@ export function applyExpectedOutcomeChange(testCase, change) {
     if (!target) {
         return testCase;
     }
+    const commit = (updatedField) => {
+        expectedOutcome[index] = updatedField;
+        return { ...testCase, expectedOutcome };
+    };
     switch (change.operation) {
         case 'set-value': {
-            if (target.type === 'chips-input') {
+            if (isChipsInputField(target)) {
                 return testCase;
             }
-            if (target.type === 'textarea' && target.outcomeMode === 'dynamic') {
+            if (isDynamicTextareaField(target)) {
                 return testCase;
             }
-            expectedOutcome[index] = {
+            return commit({
                 ...target,
                 value: change.value,
-            };
-            return { ...testCase, expectedOutcome };
+            });
         }
         case 'add-chip': {
-            if (target.type !== 'chips-input') {
+            if (!isChipsInputField(target)) {
                 return testCase;
             }
-            expectedOutcome[index] = {
+            return commit({
                 ...target,
                 value: [...target.value, change.value],
-            };
-            return { ...testCase, expectedOutcome };
+            });
         }
         case 'remove-chip': {
-            if (target.type !== 'chips-input') {
+            if (!isChipsInputField(target)) {
                 return testCase;
             }
-            expectedOutcome[index] = {
+            return commit({
                 ...target,
                 value: target.value.filter(chip => chip !== change.value),
-            };
-            return { ...testCase, expectedOutcome };
+            });
         }
         case 'set-evaluation-approach':
             return updateExpectedOutcomeFieldApproach(testCase, index, change.value);
         case 'set-outcome-mode': {
-            if (target.type !== 'textarea') {
+            if (!isTextareaField(target)) {
                 return testCase;
             }
             const mode = change.value;
             if (mode === 'static') {
                 const { resolutionQuery: _, ...rest } = target;
-                expectedOutcome[index] = {
+                return commit({
                     ...rest,
                     outcomeMode: 'static',
                     value: '',
-                };
+                });
             }
             else {
-                expectedOutcome[index] = {
+                return commit({
                     ...target,
                     outcomeMode: 'dynamic',
                     value: '',
-                };
+                });
             }
-            return { ...testCase, expectedOutcome };
         }
         case 'set-resolution-query': {
-            if (target.type !== 'textarea' || target.outcomeMode !== 'dynamic') {
+            if (!isDynamicTextareaField(target)) {
                 return testCase;
             }
-            expectedOutcome[index] = {
+            return commit({
                 ...target,
                 resolutionQuery: change.value,
-            };
-            return { ...testCase, expectedOutcome };
+            });
+        }
+        case 'set-evaluation-source-type': {
+            if (change.value === 'text') {
+                return commit({
+                    ...target,
+                    evaluationSource: { type: 'text' },
+                });
+            }
+            const extractorId = target.evaluationSource?.type === 'custom'
+                ? target.evaluationSource.extractorId
+                : (change.fallbackExtractorId ?? '');
+            return commit({
+                ...target,
+                evaluationSource: {
+                    type: 'custom',
+                    extractorId,
+                },
+            });
+        }
+        case 'set-evaluation-source-extractor': {
+            return commit({
+                ...target,
+                evaluationSource: {
+                    type: 'custom',
+                    extractorId: change.value,
+                },
+            });
         }
     }
 }

package/dist/collection/lib/test-cases/test-case-mutations.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"test-case-mutations.js","sourceRoot":"","sources":["../../../src/lib/test-cases/test-case-mutations.ts"],"names":[],"mappings":"~~AAKA~~,OAAO,EAAE,qCAAqC,EAAE,MAAM,yCAAyC,CAAC;~~AAkChG~~,MAAM,UAAU,0BAA0B,CACxC,QAAkB,EAClB,MAA6B;IAE7B,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IACzB,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,CAAC;IAC9D,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IAEtC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,QAAQ,MAAM,CAAC,SAAS,EAAE,CAAC;QACzB,KAAK,WAAW,CAAC,CAAC,CAAC;YACjB,IAAI,~~MAAM~~,CAAC,~~IAAI~~,~~KAAK~~,~~aAAa,~~EAAE,CAAC;~~gBAClC~~,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,IAAI,~~MAAM~~,CAAC,~~IAAI,KAAK,UAAU,IAAI,~~MAAM,CAAC,~~WAAW,KAAK,SAAS,~~EAAE,CAAC;~~gBACnE~~,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,~~eAAe~~,~~CAAC~~,~~KAAK,~~CAAC~~,GAAG~~;~~gBACvB~~,GAAG,MAAM;gBACT,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC~~;YACF~~,~~OAAO,EAAE,GAAG,QAAQ,EAAE,eAAe,EAAE,~~CAAC;~~QAC1C~~,CAAC;QACD,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,IAAI,~~MAAM~~,CAAC,~~IAAI~~,~~KAAK~~,~~aAAa,~~EAAE,CAAC;~~gBAClC~~,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,~~eAAe~~,~~CAAC~~,~~KAAK,~~CAAC~~,GAAG~~;~~gBACvB~~,GAAG,MAAM;gBACT,KAAK,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;aACvC,CAAC~~;YACF~~,~~OAAO,EAAE,GAAG,QAAQ,EAAE,eAAe,EAAE,~~CAAC;~~QAC1C~~,CAAC;QACD,KAAK,aAAa,CAAC,CAAC,CAAC;YACnB,IAAI,~~MAAM~~,CAAC,~~IAAI~~,~~KAAK~~,~~aAAa,~~EAAE,CAAC;~~gBAClC~~,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,~~eAAe~~,~~CAAC~~,~~KAAK,~~CAAC~~,GAAG~~;~~gBACvB~~,GAAG,MAAM;gBACT,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,MAAM,CAAC,KAAK,CAAC;aAC1D,CAAC~~;YACF~~,~~OAAO,EAAE,GAAG,QAAQ,EAAE,eAAe,EAAE,~~CAAC;~~QAC1C~~,CAAC;QACD,KAAK,yBAAyB;YAC5B,OAAO,kCAAkC,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3E,KAAK,kBAAkB,CAAC,CAAC,CAAC;YACxB,IAAI,~~MAAM~~,CAAC,~~IAAI~~,~~KAAK~~,~~UAAU,~~EAAE,CAAC;~~gBAC/B~~,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC;YAC1B,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACtB,MAAM,EAAE,eAAe,EAAE,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC;gBAC/C,~~eAAe~~,~~CAAC~~,~~KAAK,~~CAAC~~,GAAG~~;~~oBACvB~~,GAAG,IAAI;oBACP,WAAW,EAAE,QAAQ;oBACrB,KAAK,EAAE,EAAE;iBACV,CAAC;~~YACJ~~,CAAC;iBAAM,CAAC;gBACN,~~eAAe~~,~~CAAC~~,~~KAAK,~~CAAC~~,GAAG~~;~~oBACvB~~,GAAG,MAAM;oBACT,WAAW,EAAE,SAAS;oBACtB,KAAK,EAAE,EAAE;iBACV,CAAC;~~YACJ~~,CAAC;YACD,OAAO,~~EAAE~~,GAAG,~~QAAQ~~,~~EAAE,~~eAAe,EAAE,CAAC;~~QAC1C~~,CAAC;QACD,KAAK,~~sBAAsB~~,CAAC,CAAC,CAAC;~~YAC5B~~,IAAI,MAAM,CAAC,~~IAAI~~,KAAK,~~UAAU~~,~~IAAI~~,MAAM,CAAC,~~WAAW~~,~~KAAK~~,~~SAAS~~,EAAE,CAAC;~~gBACnE~~,~~OAAO~~,~~QAAQ~~,CAAC;~~YAClB~~,CAAC;~~YACD~~,~~eAAe~~,CAAC,~~KAAK~~,CAAC,~~GAAG~~;~~gBACvB~~,GAAG,MAAM;gBACT,~~eAAe~~,EAAE,~~MAAM~~,CAAC,~~KAAK~~;~~aAC9B~~,CAAC;~~YACF~~,OAAO,~~EAAE~~,GAAG,~~QAAQ~~,EAAE,~~eAAe~~,EAAE,CAAC;~~QAC1C~~,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kCAAkC,CAChD,QAAkB,EAClB,UAAkB,EAClB,QAA4B;IAE5B,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,CAAC;IAC9D,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,CAAC,CAAC;IAE3C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,MAAM,2BAA2B,GAAG,MAAM,CAAC,oBAAoB,CAAC;IAChE,eAAe,CAAC,UAAU,CAAC,GAAG;QAC5B,GAAG,MAAM;QACT,oBAAoB,EAAE,qCAAqC,CAAC,MAAM,CAAC,IAAI,EAAE;YACvE,GAAG,2BAA2B;YAC9B,QAAQ;SACT,CAAC;KACH,CAAC;IAEF,OAAO;QACL,GAAG,QAAQ;QACX,eAAe;KAChB,CAAC;AACJ,CAAC","sourcesContent":["import {\n TestCase,\n type ExpectedOutcomeMode,\n} from '../../types/llm-test-runner';\nimport { EvaluationApproach } from '../evaluation/constants';\nimport { normalizeEvaluationParametersForField } from '../evaluation/field-evaluation-approach';\n\nexport type ExpectedOutcomeChange =\n \| {\n index: number;\n operation: 'set-value';\n value: string;\n }\n \| {\n index: number;\n operation: 'add-chip';\n value: string;\n }\n \| {\n index: number;\n operation: 'remove-chip';\n value: string;\n }\n \| {\n index: number;\n operation: 'set-evaluation-approach';\n value: EvaluationApproach;\n }\n \| {\n index: number;\n operation: 'set-outcome-mode';\n value: ExpectedOutcomeMode;\n }\n \| {\n index: number;\n operation: 'set-resolution-query';\n value: string;\n };\n\nexport function applyExpectedOutcomeChange(\n testCase: TestCase,\n change: ExpectedOutcomeChange,\n): TestCase {\n const { index } = change;\n const expectedOutcome = [...(testCase.expectedOutcome \|\| [])];\n const target = expectedOutcome[index];\n\n if (!target) {\n return testCase;\n }\n\n switch (change.operation) {\n case 'set-value': {\n if (target~~.type === 'chips-input'~~) {\n return testCase;\n }\n if (target~~.type === 'textarea' && target.outcomeMode === 'dynamic'~~) {\n return testCase;\n }\n ~~expectedOutcome[index]~~ = {\n ...target,\n value: change.value,\n };\n ~~return { ...testCase, expectedOutcome };\n~~ }\n case 'add-chip': {\n if (target~~.type !== 'chips-input'~~) {\n return testCase;\n }\n ~~expectedOutcome[index]~~ = {\n ...target,\n value: [...target.value, change.value],\n };\n ~~return { ...testCase, expectedOutcome };\n~~ }\n case 'remove-chip': {\n if (target~~.type !== 'chips-input'~~) {\n return testCase;\n }\n ~~expectedOutcome[index]~~ = {\n ...target,\n value: target.value.filter(chip => chip !== change.value),\n };\n ~~return { ...testCase, expectedOutcome };\n~~ }\n case 'set-evaluation-approach':\n return updateExpectedOutcomeFieldApproach(testCase, index, change.value);\n case 'set-outcome-mode': {\n if (target~~.type !== 'textarea'~~) {\n return testCase;\n }\n const mode = change.value;\n if (mode === 'static') {\n const { resolutionQuery: _, ...rest } = target;\n ~~expectedOutcome[index]~~ = {\n ...rest,\n outcomeMode: 'static',\n value: '',\n };\n } else {\n ~~expectedOutcome[index]~~ = {\n ...target,\n outcomeMode: 'dynamic',\n value: '',\n };\n }\n ~~return~~ { ~~...~~testCase, ~~expectedOutcome~~ };\n }\n case 'set-~~resolution~~-~~query~~': {\n if (~~target~~.~~type~~ ~~!==~~ '~~textarea~~' \|\| target~~.outcomeMode~~ ~~!==~~ '~~dynamic~~') {\n ~~return testCase~~;\n }\n ~~expectedOutcome[index]~~ = {\n ...target,\n ~~resolutionQuery~~: ~~change.value~~,\n };\n return { ...~~testCase,~~ ~~expectedOutcome~~ };\n }\n }\n}\n\n/*\n Updates the evaluation approach for a specific expected outcome field.\n * Select fields always use exact matching.\n */\nexport function updateExpectedOutcomeFieldApproach(\n testCase: TestCase,\n fieldIndex: number,\n approach: EvaluationApproach,\n): TestCase {\n const expectedOutcome = [...(testCase.expectedOutcome \|\| [])];\n const target = expectedOutcome[fieldIndex];\n\n if (!target) {\n return testCase;\n }\n\n const currentEvaluationParameters = target.evaluationParameters;\n expectedOutcome[fieldIndex] = {\n ...target,\n evaluationParameters: normalizeEvaluationParametersForField(target.type, {\n ...currentEvaluationParameters,\n approach,\n }),\n };\n\n return {\n ...testCase,\n expectedOutcome,\n };\n}\n"]}
1	+ {"version":3,"file":"test-case-mutations.js","sourceRoot":"","sources":["../../../src/lib/test-cases/test-case-mutations.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,qCAAqC,EAAE,MAAM,yCAAyC,CAAC;AAEhG,SAAS,iBAAiB,CACxB,KAA2B;IAE3B,OAAO,KAAK,CAAC,IAAI,KAAK,aAAa,CAAC;AACtC,CAAC;AAED,SAAS,eAAe,CACtB,KAA2B;IAE3B,OAAO,KAAK,CAAC,IAAI,KAAK,UAAU,CAAC;AACnC,CAAC;AAED,SAAS,sBAAsB,CAC7B,KAA2B;IAE3B,OAAO,eAAe,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC;AACnE,CAAC;AA8CD,MAAM,UAAU,0BAA0B,CACxC,QAAkB,EAClB,MAA6B;IAE7B,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IACzB,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,CAAC;IAC9D,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IAEtC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,CAAC,YAAkC,EAAY,EAAE;QAC9D,eAAe,CAAC,KAAK,CAAC,GAAG,YAAY,CAAC;QACtC,OAAO,EAAE,GAAG,QAAQ,EAAE,eAAe,EAAE,CAAC;IAC1C,CAAC,CAAC;IAEF,QAAQ,MAAM,CAAC,SAAS,EAAE,CAAC;QACzB,KAAK,WAAW,CAAC,CAAC,CAAC;YACjB,IAAI,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9B,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,IAAI,sBAAsB,CAAC,MAAM,CAAC,EAAE,CAAC;gBACnC,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,OAAO,MAAM,CAAC;gBACZ,GAAG,MAAM;gBACT,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC,CAAC;QACL,CAAC;QACD,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC/B,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,OAAO,MAAM,CAAC;gBACZ,GAAG,MAAM;gBACT,KAAK,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;aACvC,CAAC,CAAC;QACL,CAAC;QACD,KAAK,aAAa,CAAC,CAAC,CAAC;YACnB,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC/B,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,OAAO,MAAM,CAAC;gBACZ,GAAG,MAAM;gBACT,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,MAAM,CAAC,KAAK,CAAC;aAC1D,CAAC,CAAC;QACL,CAAC;QACD,KAAK,yBAAyB;YAC5B,OAAO,kCAAkC,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3E,KAAK,kBAAkB,CAAC,CAAC,CAAC;YACxB,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7B,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC;YAC1B,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACtB,MAAM,EAAE,eAAe,EAAE,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC;gBAC/C,OAAO,MAAM,CAAC;oBACZ,GAAG,IAAI;oBACP,WAAW,EAAE,QAAQ;oBACrB,KAAK,EAAE,EAAE;iBACV,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,OAAO,MAAM,CAAC;oBACZ,GAAG,MAAM;oBACT,WAAW,EAAE,SAAS;oBACtB,KAAK,EAAE,EAAE;iBACV,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,KAAK,sBAAsB,CAAC,CAAC,CAAC;YAC5B,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,EAAE,CAAC;gBACpC,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,OAAO,MAAM,CAAC;gBACZ,GAAG,MAAM;gBACT,eAAe,EAAE,MAAM,CAAC,KAAK;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,KAAK,4BAA4B,CAAC,CAAC,CAAC;YAClC,IAAI,MAAM,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;gBAC5B,OAAO,MAAM,CAAC;oBACZ,GAAG,MAAM;oBACT,gBAAgB,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;iBACnC,CAAC,CAAC;YACL,CAAC;YAED,MAAM,WAAW,GACf,MAAM,CAAC,gBAAgB,EAAE,IAAI,KAAK,QAAQ;gBACxC,CAAC,CAAC,MAAM,CAAC,gBAAgB,CAAC,WAAW;gBACrC,CAAC,CAAC,CAAC,MAAM,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC;YACzC,OAAO,MAAM,CAAC;gBACZ,GAAG,MAAM;gBACT,gBAAgB,EAAE;oBAChB,IAAI,EAAE,QAAQ;oBACd,WAAW;iBACZ;aACF,CAAC,CAAC;QACL,CAAC;QACD,KAAK,iCAAiC,CAAC,CAAC,CAAC;YACvC,OAAO,MAAM,CAAC;gBACZ,GAAG,MAAM;gBACT,gBAAgB,EAAE;oBAChB,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,MAAM,CAAC,KAAK;iBAC1B;aACF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kCAAkC,CAChD,QAAkB,EAClB,UAAkB,EAClB,QAA4B;IAE5B,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,CAAC;IAC9D,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,CAAC,CAAC;IAE3C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,MAAM,2BAA2B,GAAG,MAAM,CAAC,oBAAoB,CAAC;IAChE,eAAe,CAAC,UAAU,CAAC,GAAG;QAC5B,GAAG,MAAM;QACT,oBAAoB,EAAE,qCAAqC,CAAC,MAAM,CAAC,IAAI,EAAE;YACvE,GAAG,2BAA2B;YAC9B,QAAQ;SACT,CAAC;KACH,CAAC;IAEF,OAAO;QACL,GAAG,QAAQ;QACX,eAAe;KAChB,CAAC;AACJ,CAAC","sourcesContent":["import {\n TestCase,\n type ExpectedOutcomeField,\n type EvaluationSource,\n type ExpectedOutcomeMode,\n} from '../../types/llm-test-runner';\nimport { EvaluationApproach } from '../evaluation/constants';\nimport { normalizeEvaluationParametersForField } from '../evaluation/field-evaluation-approach';\n\nfunction isChipsInputField(\n field: ExpectedOutcomeField,\n): field is Extract<ExpectedOutcomeField, { type: 'chips-input' }> {\n return field.type === 'chips-input';\n}\n\nfunction isTextareaField(\n field: ExpectedOutcomeField,\n): field is Extract<ExpectedOutcomeField, { type: 'textarea' }> {\n return field.type === 'textarea';\n}\n\nfunction isDynamicTextareaField(\n field: ExpectedOutcomeField,\n): field is Extract<ExpectedOutcomeField, { type: 'textarea' }> {\n return isTextareaField(field) && field.outcomeMode === 'dynamic';\n}\n\n\nexport type ExpectedOutcomeChange =\n \| {\n index: number;\n operation: 'set-value';\n value: string;\n }\n \| {\n index: number;\n operation: 'add-chip';\n value: string;\n }\n \| {\n index: number;\n operation: 'remove-chip';\n value: string;\n }\n \| {\n index: number;\n operation: 'set-evaluation-approach';\n value: EvaluationApproach;\n }\n \| {\n index: number;\n operation: 'set-outcome-mode';\n value: ExpectedOutcomeMode;\n }\n \| {\n index: number;\n operation: 'set-resolution-query';\n value: string;\n }\n \| {\n index: number;\n operation: 'set-evaluation-source-type';\n value: EvaluationSource['type'];\n fallbackExtractorId?: string;\n }\n \| {\n index: number;\n operation: 'set-evaluation-source-extractor';\n value: string;\n };\n\nexport function applyExpectedOutcomeChange(\n testCase: TestCase,\n change: ExpectedOutcomeChange,\n): TestCase {\n const { index } = change;\n const expectedOutcome = [...(testCase.expectedOutcome \|\| [])];\n const target = expectedOutcome[index];\n\n if (!target) {\n return testCase;\n }\n\n const commit = (updatedField: ExpectedOutcomeField): TestCase => {\n expectedOutcome[index] = updatedField;\n return { ...testCase, expectedOutcome };\n };\n\n switch (change.operation) {\n case 'set-value': {\n if (isChipsInputField(target)) {\n return testCase;\n }\n if (isDynamicTextareaField(target)) {\n return testCase;\n }\n return commit({\n ...target,\n value: change.value,\n });\n }\n case 'add-chip': {\n if (!isChipsInputField(target)) {\n return testCase;\n }\n return commit({\n ...target,\n value: [...target.value, change.value],\n });\n }\n case 'remove-chip': {\n if (!isChipsInputField(target)) {\n return testCase;\n }\n return commit({\n ...target,\n value: target.value.filter(chip => chip !== change.value),\n });\n }\n case 'set-evaluation-approach':\n return updateExpectedOutcomeFieldApproach(testCase, index, change.value);\n case 'set-outcome-mode': {\n if (!isTextareaField(target)) {\n return testCase;\n }\n const mode = change.value;\n if (mode === 'static') {\n const { resolutionQuery: _, ...rest } = target;\n return commit({\n ...rest,\n outcomeMode: 'static',\n value: '',\n });\n } else {\n return commit({\n ...target,\n outcomeMode: 'dynamic',\n value: '',\n });\n }\n }\n case 'set-resolution-query': {\n if (!isDynamicTextareaField(target)) {\n return testCase;\n }\n return commit({\n ...target,\n resolutionQuery: change.value,\n });\n }\n case 'set-evaluation-source-type': {\n if (change.value === 'text') {\n return commit({\n ...target,\n evaluationSource: { type: 'text' },\n });\n }\n\n const extractorId =\n target.evaluationSource?.type === 'custom'\n ? target.evaluationSource.extractorId\n : (change.fallbackExtractorId ?? '');\n return commit({\n ...target,\n evaluationSource: {\n type: 'custom',\n extractorId,\n },\n });\n }\n case 'set-evaluation-source-extractor': {\n return commit({\n ...target,\n evaluationSource: {\n type: 'custom',\n extractorId: change.value,\n },\n });\n }\n }\n}\n\n/*\n Updates the evaluation approach for a specific expected outcome field.\n * Select fields always use exact matching.\n */\nexport function updateExpectedOutcomeFieldApproach(\n testCase: TestCase,\n fieldIndex: number,\n approach: EvaluationApproach,\n): TestCase {\n const expectedOutcome = [...(testCase.expectedOutcome \|\| [])];\n const target = expectedOutcome[fieldIndex];\n\n if (!target) {\n return testCase;\n }\n\n const currentEvaluationParameters = target.evaluationParameters;\n expectedOutcome[fieldIndex] = {\n ...target,\n evaluationParameters: normalizeEvaluationParametersForField(target.type, {\n ...currentEvaluationParameters,\n approach,\n }),\n };\n\n return {\n ...testCase,\n expectedOutcome,\n };\n}\n"]}

package/dist/collection/schemas/expected-outcome.js CHANGED Viewed

@@ -6,6 +6,19 @@ const optionalPositiveInt = z.number().int().positive().optional();
 const optionalString = z.string().optional();
 const selectOptionsSchema = z.array(nonEmptyString).min(1);
 const optionalNumber = z.number().optional();
+const textEvaluationSourceSchema = z.object({
+    type: z.literal('text'),
+});
+const customEvaluationSourceSchema = z.object({
+    type: z.literal('custom'),
+    extractorId: nonEmptyString,
+});
+export const evaluationSourceExtractorSchema = z.custom(value => typeof value === 'function', 'Extractor must be a function.');
+export const evaluationSourceExtractorsSchema = z.record(z.string().min(1), evaluationSourceExtractorSchema);
+export const evaluationSourceSchema = z.discriminatedUnion('type', [
+    textEvaluationSourceSchema,
+    customEvaluationSourceSchema,
+]);
 export const expectedOutcomeModeSchema = z.enum(['static', 'dynamic']);
 const evaluationParametersSchema = z.object({
     approach: z.enum(EvaluationApproach),
@@ -23,6 +36,7 @@ const selectEvaluationParametersSchema = evaluationParametersSchema.superRefine(
 const defaultExpectedOutcomeBaseSchema = z.object({
     label: nonEmptyString,
     placeholder: optionalString,
+    evaluationSource: evaluationSourceSchema.optional(),
 });
 const createDefaultExpectedOutcomeFieldSchemas = (baseSchema) => ({
     text: baseSchema.extend({
@@ -122,4 +136,29 @@ export function validateExpectedOutcomeArray(expectedOutcome) {
         throw new Error(`Invalid expectedOutcome: ${parsed.error.issues[0].message}`);
     }
 }
+export function validateExpectedOutcomeArrayWithExtractors(expectedOutcome, allowedExtractorIds) {
+    const allowed = new Set(allowedExtractorIds);
+    const schema = expectedOutcomeArraySchema.superRefine((fields, ctx) => {
+        fields.forEach((field, index) => {
+            if (field.evaluationSource?.type !== 'custom') {
+                return;
+            }
+            if (allowed.has(field.evaluationSource.extractorId)) {
+                return;
+            }
+            ctx.addIssue({
+                code: 'custom',
+                path: [index, 'evaluationSource', 'extractorId'],
+                message: `Invalid expectedOutcome: Extractor "${field.evaluationSource.extractorId}" is not registered.`,
+            });
+        });
+    });
+    const parsed = schema.safeParse(expectedOutcome);
+    if (!parsed.success) {
+        throw new Error(parsed.error.issues[0].message);
+    }
+}
+export function getExtractorIds(extractors) {
+    return Object.keys(extractors || {});
+}
 //# sourceMappingURL=expected-outcome.js.map