llm-testrunner-components 1.1.0 → 1.2.0

package/dist/collection/lib/evaluation/rouge1-evaluator.test.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"rouge1-evaluator.test.js","sourceRoot":"","sources":["../../../src/lib/evaluation/rouge1-evaluator.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAErD,OAAO,EAAE,wBAAwB,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAC3E,mEAAmE;AACnE,8FAA8F;AAC9F,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AAExE,MAAM,WAAW,GAAsB;IACrC,UAAU,EAAE,UAAU;IACtB,QAAQ,EAAE,oBAAoB;IAC9B,cAAc,EAAE,6BAA6B;IAC7C,eAAe,EAAE,iBAAiB;IAClC,oBAAoB,EAAE;QACpB,QAAQ,EAAE,kBAAkB,CAAC,OAAO;QACpC,SAAS,EAAE,GAAG;KACf;CACF,CAAC;AAEF,MAAM,sBAAsB,GAAsB;IAChD,GAAG,WAAW;IACd,oBAAoB,EAAE;QACpB,QAAQ,EAAE,kBAAkB,CAAC,OAAO;QACpC,SAAS,EAAE,SAAS;KACrB;CACF,CAAC;AAEF,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;YACxE,MAAM,OAAO,GAAsB;gBACjC,GAAG,WAAW;gBACd,cAAc,EAAE,iCAAiC;gBACjD,eAAe,EAAE,iBAAiB;aACnC,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEtD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC7C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,CACJ,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,KAAK,CACxD,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;YACvB,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,CACJ,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,KAAK,CACxD,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;YACtE,MAAM,OAAO,GAAsB;gBACjC,GAAG,WAAW;gBACd,cAAc,EAAE,oDAAoD;gBACpE,eAAe,EAAE,2CAA2C;aAC7D,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEtD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnD,MAAM,CACJ,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,KAAK,CACxD,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;YACpB,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnD,MAAM,CACJ,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,KAAK,CACxD,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,OAAO,GAAsB;gBACjC,GAAG,WAAW;gBACd,cAAc,EAAE,iCAAiC;gBACjD,eAAe,EAAE,oCAAoC;aACtD,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEtD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,CACJ,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,KAAK,CACxD,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;YAC9B,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnD,MAAM,CACJ,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,KAAK,CACxD,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC9D,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,sBAAsB,CAAC,CAAC;YAErE,MAAM,CAAC,MAAM,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC,IAAI,CAChD,wBAAwB,CACzB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,OAAO,GAAsB;gBACjC,GAAG,WAAW;gBACd,cAAc,EAAE,yCAAyC;gBACzD,eAAe,EAAE,8BAA8B;gBAC/C,oBAAoB,EAAE;oBACpB,QAAQ,EAAE,kBAAkB,CAAC,OAAO;oBACpC,SAAS,EAAE,GAAG;iBACf;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEtD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7D,MAAM,CAAC,MAAM,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,OAAO,GAAsB;gBACjC,GAAG,WAAW;gBACd,cAAc,EAAE,iCAAiC;gBACjD,eAAe,EAAE,kBAAkB;gBACnC,oBAAoB,EAAE;oBACpB,QAAQ,EAAE,kBAAkB,CAAC,OAAO;oBACpC,SAAS,EAAE,GAAG;iBACf;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEtD,MAAM,CAAC,MAAM,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxD,MAAM,CACJ,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,KAAK,CACxD,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;QAC1B,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,OAAO,GAAG,EAAE,GAAG,WAAW,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC;YAEvD,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEtD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACxE,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,MAAM,OAAO,GAAG,EAAE,GAAG,WAAW,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC;YAExD,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEtD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC","sourcesContent":["import { describe, it, expect } from '@jest/globals';\nimport { EvaluationRequest } from './types';\nimport { DEFAULT_ROUGE_PASS_SCORE, EvaluationApproach } from './constants';\n// Using integration tests with actual js-rouge library (no mocks).\n// This approach tests the real ROUGE-1 scoring behavior rather than just orchestration logic.\nimport { performRouge1Evaluation } from './evaluators/rouge1-evaluator';\n\nconst mockRequest: EvaluationRequest = {\n  testCaseId: 'test-000',\n  question: 'What is your name?',\n  actualResponse: 'I am a large language model',\n  expectedOutcome: 'model\\nlanguage',\n  evaluationParameters: {\n    approach: EvaluationApproach.ROUGE_1,\n    threshold: 0.5,\n  },\n};\n\nconst mockRequestNoThreshold: EvaluationRequest = {\n  ...mockRequest,\n  evaluationParameters: {\n    approach: EvaluationApproach.ROUGE_1,\n    threshold: undefined,\n  },\n};\n\ndescribe('performRouge1Evaluation', () => {\n  describe('Basic functionality', () => {\n    it('should pass when response contains exact keyword matches', async () => {\n      const request: EvaluationRequest = {\n        ...mockRequest,\n        actualResponse: 'This is a language model system',\n        expectedOutcome: 'language\\nmodel',\n      };\n\n      const result = await performRouge1Evaluation(request);\n\n      expect(result.passed).toBe(true);\n      expect(result.keywordMatches.length).toBe(2);\n      expect(result.keywordMatches[0].found).toBe(true);\n      expect(\n        result.keywordMatches[0].evaluationApproachResult.score,\n      ).toBeGreaterThan(0.5);\n      expect(result.keywordMatches[1].found).toBe(true);\n      expect(\n        result.keywordMatches[1].evaluationApproachResult.score,\n      ).toBeGreaterThan(0.5);\n    });\n\n    it('should fail when keywords are not sufficiently present', async () => {\n      const request: EvaluationRequest = {\n        ...mockRequest,\n        actualResponse: 'This is completely unrelated content about cooking',\n        expectedOutcome: 'machine learning\\nartificial intelligence',\n      };\n\n      const result = await performRouge1Evaluation(request);\n\n      expect(result.passed).toBe(false);\n      expect(result.keywordMatches[0].found).toBe(false);\n      expect(\n        result.keywordMatches[0].evaluationApproachResult.score,\n      ).toBeLessThan(0.5);\n      expect(result.keywordMatches[1].found).toBe(false);\n      expect(\n        result.keywordMatches[1].evaluationApproachResult.score,\n      ).toBeLessThan(0.5);\n    });\n\n    it('should partially pass when only some keywords meet threshold', async () => {\n      const request: EvaluationRequest = {\n        ...mockRequest,\n        actualResponse: 'Machine learning is fascinating',\n        expectedOutcome: 'machine learning\\ndatabase systems',\n      };\n\n      const result = await performRouge1Evaluation(request);\n\n      expect(result.passed).toBe(false);\n      expect(result.keywordMatches[0].found).toBe(true);\n      expect(\n        result.keywordMatches[0].evaluationApproachResult.score,\n      ).toBeGreaterThanOrEqual(0.5);\n      expect(result.keywordMatches[1].found).toBe(false);\n      expect(\n        result.keywordMatches[1].evaluationApproachResult.score,\n      ).toBeLessThan(0.5);\n    });\n  });\n\n  describe('Threshold handling', () => {\n    it('should use default threshold when not provided', async () => {\n      const result = await performRouge1Evaluation(mockRequestNoThreshold);\n\n      expect(result.evaluationParameters.threshold).toBe(\n        DEFAULT_ROUGE_PASS_SCORE,\n      );\n    });\n\n    it('should pass all keywords with threshold 0.0', async () => {\n      const request: EvaluationRequest = {\n        ...mockRequest,\n        actualResponse: 'completely unrelated text about cooking',\n        expectedOutcome: 'quantum physics\\nmathematics',\n        evaluationParameters: {\n          approach: EvaluationApproach.ROUGE_1,\n          threshold: 0.0,\n        },\n      };\n\n      const result = await performRouge1Evaluation(request);\n\n      expect(result.passed).toBe(true);\n      expect(result.keywordMatches.every(m => m.found)).toBe(true);\n      expect(result.evaluationParameters.threshold).toBe(0.0);\n    });\n\n    it('should fail when threshold is 1.0 and match is not perfect', async () => {\n      const request: EvaluationRequest = {\n        ...mockRequest,\n        actualResponse: 'This is about learning concepts',\n        expectedOutcome: 'machine learning',\n        evaluationParameters: {\n          approach: EvaluationApproach.ROUGE_1,\n          threshold: 1.0,\n        },\n      };\n\n      const result = await performRouge1Evaluation(request);\n\n      expect(result.evaluationParameters.threshold).toBe(1.0);\n      expect(\n        result.keywordMatches[0].evaluationApproachResult.score,\n      ).toBeLessThan(1.0);\n    });\n  });\n\n  describe('Edge cases', () => {\n    it('should handle empty actualResponse', async () => {\n      const request = { ...mockRequest, actualResponse: '' };\n\n      const result = await performRouge1Evaluation(request);\n\n      expect(result.passed).toBe(false);\n      expect(result.keywordMatches[0].evaluationApproachResult.score).toBe(0);\n      expect(result.keywordMatches[1].evaluationApproachResult.score).toBe(0);\n    });\n\n    it('should handle empty expectedOutcome string', async () => {\n      const request = { ...mockRequest, expectedOutcome: '' };\n\n      const result = await performRouge1Evaluation(request);\n\n      expect(result.passed).toBe(true);\n      expect(result.keywordMatches.length).toBe(0);\n    });\n  });\n});\n"]}