llm-testrunner-components 1.0.5 → 1.0.7

package/dist/collection/lib/evaluation/evaluators/rouge1-evaluator.js

@@ -0,0 +1,82 @@
+import * as rouge from "js-rouge";
+import { DEFAULT_ROUGE_PASS_SCORE, ROUGE } from "../constant";
+/**
+ * Evaluates a single keyword against the candidate text using ROUGE-1.
+ *
+ * @param {string} keyword - The expected keyword to evaluate
+ * @param {string} candidate - The actual response text
+ * @param {number} rougeThreshold - The minimum ROUGE-1 score required to pass
+ * @returns {KeywordMatch} The evaluation result for this keyword
+ */
+function evaluateKeyword(keyword, candidate, rougeThreshold) {
+    let rouge1Score = 0;
+    try {
+        if (keyword.trim().length > 0 && candidate.length > 0) {
+            const rouge1 = rouge.n(candidate, keyword.trim(), { n: 1 });
+            rouge1Score = isNaN(rouge1) ? 0 : rouge1;
+        }
+        else {
+            console.warn(`ROUGE-1 not computed for keyword "${keyword}": Keyword or Candidate is missing.`);
+        }
+    }
+    catch (err) {
+        console.error(`ROUGE-1 computation failed for keyword "${keyword}":`, err);
+    }
+    const keywordPassed = rouge1Score >= rougeThreshold;
+    const keywordApproachResult = {
+        score: rouge1Score,
+        approachUsed: ROUGE
+    };
+    return {
+        keyword: keyword,
+        found: keywordPassed,
+        evaluationApproachResult: keywordApproachResult
+    };
+}
+/**
+ * Computes the ROUGE-1 score for a single keyword against the candidate text.
+ *
+ * ROUGE-1 measures the overlap of unigrams (single words) between the candidate
+ * and reference text. A score of 1.0 indicates perfect overlap.
+ *
+ * @example
+ * const match = evaluateSingleKeyword(
+ *   "The quick brown fox",
+ *   "quick fox",
+ *   0.5
+ * );
+ * // Returns: { keyword: "quick fox", found: true, score: 0.67, ... }
+ * //general idea , here we are doing it. by word to word comparison
+ */
+export async function performRouge1Evaluation(request) {
+    const { testCaseId, actualResponse, expectedKeywords, evaluationParameters } = request;
+    const candidate = (actualResponse || '').trim();
+    const rougeThreshold = evaluationParameters.threshold ?? DEFAULT_ROUGE_PASS_SCORE;
+    let keywordsPassed = 0;
+    const totalKeywords = expectedKeywords.length;
+    const keywordMatches = expectedKeywords.map(keyword => {
+        const match = evaluateKeyword(keyword, candidate, rougeThreshold);
+        if (match.found) {
+            keywordsPassed++;
+        }
+        return match;
+    });
+    const overallPassed = keywordsPassed === totalKeywords;
+    const overallApproachResult = {
+        score: keywordsPassed / totalKeywords,
+        approachUsed: ROUGE
+    };
+    return {
+        testCaseId: testCaseId,
+        passed: overallPassed,
+        keywordMatches: keywordMatches,
+        sourceLinkMatches: [],
+        timestamp: new Date().toISOString(),
+        evaluationParameters: {
+            ...evaluationParameters,
+            threshold: rougeThreshold
+        },
+        evaluationApproachResult: overallApproachResult
+    };
+}
+//# sourceMappingURL=rouge1-evaluator.js.map

package/dist/collection/lib/evaluation/evaluators/rouge1-evaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rouge1-evaluator.js","sourceRoot":"","sources":["../../../../src/lib/evaluation/evaluators/rouge1-evaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,UAAU,CAAC;AAGlC,OAAO,EAAE,wBAAwB,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAE9D;;;;;;;GAOG;AACH,SAAS,eAAe,CAAC,OAAe,EAAE,SAAiB,EAAE,cAAsB;IAC/E,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,IAAI,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,SAAS,EAAE,OAAO,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YAC5D,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAE7C,CAAC;aAAM,CAAC;YACJ,OAAO,CAAC,IAAI,CAAC,qCAAqC,OAAO,qCAAqC,CAAC,CAAC;QACpG,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,2CAA2C,OAAO,IAAI,EAAE,GAAG,CAAC,CAAC;IAC/E,CAAC;IAED,MAAM,aAAa,GAAG,WAAW,IAAI,cAAc,CAAC;IAEpD,MAAM,qBAAqB,GAA6B;QACpD,KAAK,EAAE,WAAW;QAClB,YAAY,EAAE,KAAK;KACtB,CAAC;IAEF,OAAO;QACH,OAAO,EAAE,OAAO;QAChB,KAAK,EAAE,aAAa;QACpB,wBAAwB,EAAE,qBAAqB;KAClD,CAAC;AACN,CAAC;AAED;;;;;;;;;;;;;;GAcG;AAEH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,OAA0B;IACpE,MAAM,EACF,UAAU,EACV,cAAc,EACd,gBAAgB,EAChB,oBAAoB,EACvB,GAAG,OAAO,CAAC;IAEZ,MAAM,SAAS,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAChD,MAAM,cAAc,GAAG,oBAAoB,CAAC,SAAS,IAAI,wBAAwB,CAAC;IAElF,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,MAAM,aAAa,GAAG,gBAAgB,CAAC,MAAM,CAAC;IAE9C,MAAM,cAAc,GAAmB,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;QAClE,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,EAAE,SAAS,EAAE,cAAc,CAAC,CAAC;QAElE,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,cAAc,EAAE,CAAC;QACrB,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC,CAAC,CAAC;IAEH,MAAM,aAAa,GAAG,cAAc,KAAK,aAAa,CAAC;IAEvD,MAAM,qBAAqB,GAA6B;QACpD,KAAK,EAAE,cAAc,GAAG,aAAa;QACrC,YAAY,EAAE,KAAK;KACtB,CAAC;IAEF,OAAO;QACH,UAAU,EAAE,UAAU;QACtB,MAAM,EAAE,aAAa;QACrB,cAAc,EAAE,cAAc;QAC9B,iBAAiB,EAAE,EAAE;QACrB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,oBAAoB,EAAE;YAClB,GAAG,oBAAoB;YACvB,SAAS,EAAE,cAAc;SAC5B;QACD,wBAAwB,EAAE,qBAAqB;KAClD,CAAC;AACN,CAAC","sourcesContent":["import * as rouge from 'js-rouge';\nimport { EvaluationApproachResult } from '../../../types/evaluation';\nimport { EvaluationRequest, EvaluationResult, KeywordMatch } from '../types';\nimport { DEFAULT_ROUGE_PASS_SCORE, ROUGE } from '../constant';\n\n/**\n * Evaluates a single keyword against the candidate text using ROUGE-1.\n * \n * @param {string} keyword - The expected keyword to evaluate\n * @param {string} candidate - The actual response text\n * @param {number} rougeThreshold - The minimum ROUGE-1 score required to pass\n * @returns {KeywordMatch} The evaluation result for this keyword\n */\nfunction evaluateKeyword(keyword: string, candidate: string, rougeThreshold: number): KeywordMatch {\n    let rouge1Score = 0;\n\n    try {\n        if (keyword.trim().length > 0 && candidate.length > 0) {\n            const rouge1 = rouge.n(candidate, keyword.trim(), { n: 1 });\n            rouge1Score = isNaN(rouge1) ? 0 : rouge1;\n\n        } else {\n            console.warn(`ROUGE-1 not computed for keyword \"${keyword}\": Keyword or Candidate is missing.`);\n        }\n    } catch (err) {\n        console.error(`ROUGE-1 computation failed for keyword \"${keyword}\":`, err);\n    }\n\n    const keywordPassed = rouge1Score >= rougeThreshold;\n\n    const keywordApproachResult: EvaluationApproachResult = {\n        score: rouge1Score,\n        approachUsed: ROUGE\n    };\n\n    return {\n        keyword: keyword,\n        found: keywordPassed,\n        evaluationApproachResult: keywordApproachResult\n    };\n}\n\n/**\n * Computes the ROUGE-1 score for a single keyword against the candidate text.\n * \n * ROUGE-1 measures the overlap of unigrams (single words) between the candidate\n * and reference text. A score of 1.0 indicates perfect overlap.\n * \n * @example\n * const match = evaluateSingleKeyword(\n *   \"The quick brown fox\",\n *   \"quick fox\",\n *   0.5\n * );\n * // Returns: { keyword: \"quick fox\", found: true, score: 0.67, ... }\n * //general idea , here we are doing it. by word to word comparison\n */\n\nexport async function performRouge1Evaluation(request: EvaluationRequest): Promise<EvaluationResult> {\n    const {\n        testCaseId,\n        actualResponse,\n        expectedKeywords,\n        evaluationParameters\n    } = request;\n\n    const candidate = (actualResponse || '').trim();\n    const rougeThreshold = evaluationParameters.threshold ?? DEFAULT_ROUGE_PASS_SCORE;\n\n    let keywordsPassed = 0;\n    const totalKeywords = expectedKeywords.length;\n\n    const keywordMatches: KeywordMatch[] = expectedKeywords.map(keyword => {\n        const match = evaluateKeyword(keyword, candidate, rougeThreshold);\n        \n        if (match.found) {\n            keywordsPassed++;\n        }\n        \n        return match;\n    });\n\n    const overallPassed = keywordsPassed === totalKeywords;\n\n    const overallApproachResult: EvaluationApproachResult = {\n        score: keywordsPassed / totalKeywords,\n        approachUsed: ROUGE\n    };\n\n    return {\n        testCaseId: testCaseId,\n        passed: overallPassed,\n        keywordMatches: keywordMatches,\n        sourceLinkMatches: [],\n        timestamp: new Date().toISOString(),\n        evaluationParameters: {\n            ...evaluationParameters,\n            threshold: rougeThreshold\n        },\n        evaluationApproachResult: overallApproachResult\n    };\n}"]}

package/dist/collection/lib/evaluation/evaluators/rougeL-evaluator.js

@@ -0,0 +1,73 @@
+import * as rouge from "js-rouge";
+import { DEFAULT_ROUGE_PASS_SCORE, ROUGE } from "../constant";
+function evaluateKeyword(keyword, candidate, rougeThreshold) {
+    let rougeLScore = 0;
+    try {
+        const trimmedKeyword = keyword.trim();
+        if (trimmedKeyword.length > 0 && candidate.length > 0) {
+            const referenceTokens = trimmedKeyword.toLowerCase().split(/\s+/);
+            const candidateTokens = candidate.toLowerCase().split(/\s+/);
+            if (referenceTokens.length === 1 && candidateTokens.includes(referenceTokens[0])) {
+                rougeLScore = 1;
+            }
+            else {
+                const lcsResult = rouge.lcs(candidateTokens, referenceTokens);
+                const lcsLength = typeof lcsResult === 'number'
+                    ? lcsResult
+                    : (lcsResult?.length ?? 0);
+                const recall = referenceTokens.length > 0 ? lcsLength / referenceTokens.length : 0;
+                const precision = candidateTokens.length > 0 ? lcsLength / candidateTokens.length : 0;
+                const denominator = precision + recall;
+                const f1Score = denominator > 0 ? (2 * precision * recall) / denominator : 0;
+                rougeLScore = f1Score;
+            }
+        }
+        else {
+            console.warn(`ROUGE-L not computed for keyword "${keyword}": Keyword or candidate missing.`);
+        }
+    }
+    catch (err) {
+        console.error(`ROUGE-L computation failed for keyword "${keyword}":`, err);
+    }
+    const keywordPassed = rougeLScore >= rougeThreshold;
+    const keywordApproachResult = {
+        score: rougeLScore,
+        approachUsed: ROUGE,
+    };
+    return {
+        keyword,
+        found: keywordPassed,
+        evaluationApproachResult: keywordApproachResult,
+    };
+}
+export function performRougeLEvaluation(request) {
+    const { testCaseId, actualResponse, expectedKeywords, evaluationParameters, } = request;
+    const candidate = (actualResponse || '').trim();
+    const rougeThreshold = evaluationParameters.threshold ?? DEFAULT_ROUGE_PASS_SCORE;
+    let keywordsPassed = 0;
+    const totalKeywords = expectedKeywords.length;
+    const keywordMatches = expectedKeywords.map((keyword) => {
+        const match = evaluateKeyword(keyword, candidate, rougeThreshold);
+        if (match.found)
+            keywordsPassed++;
+        return match;
+    });
+    const overallPassed = keywordsPassed === totalKeywords;
+    const overallApproachResult = {
+        score: totalKeywords > 0 ? keywordsPassed / totalKeywords : 1,
+        approachUsed: ROUGE,
+    };
+    return {
+        testCaseId,
+        passed: overallPassed,
+        keywordMatches,
+        sourceLinkMatches: [],
+        timestamp: new Date().toISOString(),
+        evaluationParameters: {
+            ...evaluationParameters,
+            threshold: rougeThreshold,
+        },
+        evaluationApproachResult: overallApproachResult,
+    };
+}
+//# sourceMappingURL=rougeL-evaluator.js.map

package/dist/collection/lib/evaluation/evaluators/rougeL-evaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rougeL-evaluator.js","sourceRoot":"","sources":["../../../../src/lib/evaluation/evaluators/rougeL-evaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,UAAU,CAAC;AAGlC,OAAO,EAAE,wBAAwB,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAG9D,SAAS,eAAe,CAAC,OAAe,EAAE,SAAiB,EAAE,cAAsB;IACjF,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,IAAI,CAAC;QACH,MAAM,cAAc,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtD,MAAM,eAAe,GAAG,cAAc,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAClE,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAE7D,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,IAAI,eAAe,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjF,WAAW,GAAG,CAAC,CAAC;YAClB,CAAC;iBAAM,CAAC;gBAEN,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;gBAC9D,MAAM,SAAS,GACb,OAAO,SAAS,KAAK,QAAQ;oBAC3B,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,CAAC,SAAS,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;gBAE/B,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnF,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBACtF,MAAM,WAAW,GAAG,SAAS,GAAG,MAAM,CAAC;gBAEvC,MAAM,OAAO,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC7E,WAAW,GAAG,OAAO,CAAC;YACxB,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CAAC,qCAAqC,OAAO,kCAAkC,CAAC,CAAC;QAC/F,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,2CAA2C,OAAO,IAAI,EAAE,GAAG,CAAC,CAAC;IAC7E,CAAC;IAED,MAAM,aAAa,GAAG,WAAW,IAAI,cAAc,CAAC;IAEpD,MAAM,qBAAqB,GAA6B;QACtD,KAAK,EAAE,WAAW;QAClB,YAAY,EAAE,KAAK;KACpB,CAAC;IAEF,OAAO;QACL,OAAO;QACP,KAAK,EAAE,aAAa;QACpB,wBAAwB,EAAE,qBAAqB;KAChD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,OAA0B;IAChE,MAAM,EACJ,UAAU,EACV,cAAc,EACd,gBAAgB,EAChB,oBAAoB,GACrB,GAAG,OAAO,CAAC;IAEZ,MAAM,SAAS,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAChD,MAAM,cAAc,GAAG,oBAAoB,CAAC,SAAS,IAAI,wBAAwB,CAAC;IAElF,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,MAAM,aAAa,GAAG,gBAAgB,CAAC,MAAM,CAAC;IAE9C,MAAM,cAAc,GAAmB,gBAAgB,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;QACtE,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,EAAE,SAAS,EAAE,cAAc,CAAC,CAAC;QAClE,IAAI,KAAK,CAAC,KAAK;YAAE,cAAc,EAAE,CAAC;QAClC,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;IAEH,MAAM,aAAa,GAAG,cAAc,KAAK,aAAa,CAAC;IAEvD,MAAM,qBAAqB,GAA6B;QACtD,KAAK,EAAE,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QAC7D,YAAY,EAAE,KAAK;KACpB,CAAC;IAEF,OAAO;QACL,UAAU;QACV,MAAM,EAAE,aAAa;QACrB,cAAc;QACd,iBAAiB,EAAE,EAAE;QACrB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,oBAAoB,EAAE;YACpB,GAAG,oBAAoB;YACvB,SAAS,EAAE,cAAc;SAC1B;QACD,wBAAwB,EAAE,qBAAqB;KAChD,CAAC;AACJ,CAAC","sourcesContent":["import * as rouge from 'js-rouge';\nimport { EvaluationApproachResult } from '../../../types/evaluation';\nimport { EvaluationRequest, EvaluationResult, KeywordMatch } from '../types';\nimport { DEFAULT_ROUGE_PASS_SCORE, ROUGE } from '../constant';\n\n\nfunction evaluateKeyword(keyword: string, candidate: string, rougeThreshold: number): KeywordMatch {\n  let rougeLScore = 0;\n\n  try {\n    const trimmedKeyword = keyword.trim();\n    if (trimmedKeyword.length > 0 && candidate.length > 0) {\n      const referenceTokens = trimmedKeyword.toLowerCase().split(/\\s+/);\n      const candidateTokens = candidate.toLowerCase().split(/\\s+/);\n\n      if (referenceTokens.length === 1 && candidateTokens.includes(referenceTokens[0])) {\n        rougeLScore = 1;\n      } else {\n\n        const lcsResult = rouge.lcs(candidateTokens, referenceTokens);\n        const lcsLength =\n          typeof lcsResult === 'number'\n            ? lcsResult\n            : (lcsResult?.length ?? 0);\n\n        const recall = referenceTokens.length > 0 ? lcsLength / referenceTokens.length : 0;\n        const precision = candidateTokens.length > 0 ? lcsLength / candidateTokens.length : 0;\n        const denominator = precision + recall;\n\n        const f1Score = denominator > 0 ? (2 * precision * recall) / denominator : 0;\n        rougeLScore = f1Score;\n      }\n    } else {\n      console.warn(`ROUGE-L not computed for keyword \"${keyword}\": Keyword or candidate missing.`);\n    }\n  } catch (err) {\n    console.error(`ROUGE-L computation failed for keyword \"${keyword}\":`, err);\n  }\n\n  const keywordPassed = rougeLScore >= rougeThreshold;\n\n  const keywordApproachResult: EvaluationApproachResult = {\n    score: rougeLScore,\n    approachUsed: ROUGE,\n  };\n\n  return {\n    keyword,\n    found: keywordPassed,\n    evaluationApproachResult: keywordApproachResult,\n  };\n}\n\nexport function performRougeLEvaluation(request: EvaluationRequest): EvaluationResult {  \n  const {\n    testCaseId,\n    actualResponse,\n    expectedKeywords,\n    evaluationParameters,\n  } = request;\n\n  const candidate = (actualResponse || '').trim();\n  const rougeThreshold = evaluationParameters.threshold ?? DEFAULT_ROUGE_PASS_SCORE;\n\n  let keywordsPassed = 0;\n  const totalKeywords = expectedKeywords.length;\n\n  const keywordMatches: KeywordMatch[] = expectedKeywords.map((keyword) => {\n    const match = evaluateKeyword(keyword, candidate, rougeThreshold);\n    if (match.found) keywordsPassed++;\n    return match;\n  });\n\n  const overallPassed = keywordsPassed === totalKeywords;\n\n  const overallApproachResult: EvaluationApproachResult = {\n    score: totalKeywords > 0 ? keywordsPassed / totalKeywords : 1,\n    approachUsed: ROUGE,\n  };\n\n  return {\n    testCaseId,\n    passed: overallPassed,\n    keywordMatches,\n    sourceLinkMatches: [],\n    timestamp: new Date().toISOString(),\n    evaluationParameters: {\n      ...evaluationParameters,\n      threshold: rougeThreshold,\n    },\n    evaluationApproachResult: overallApproachResult,\n  };\n}"]}

package/dist/collection/lib/evaluation/evaluators/rougeL-evaluator.test.js

@@ -0,0 +1,313 @@
+import { jest, describe, it, expect } from "@jest/globals";
+import { performRougeLEvaluation } from "./rougeL-evaluator";
+import { DEFAULT_ROUGE_PASS_SCORE, ROUGE } from "../constant";
+describe('performRougeLEvaluation', () => {
+    // Helper function to create a base request with optional overrides
+    const createRequest = (overrides = {}) => {
+        const defaults = {
+            testCaseId: 'test-001',
+            question: 'Test question',
+            expectedKeywords: ['keyword'],
+            expectedSourceLinks: [],
+            actualResponse: 'response with keyword',
+            evaluationParameters: {
+                approach: 'rouge',
+                threshold: DEFAULT_ROUGE_PASS_SCORE,
+            },
+        };
+        return {
+            ...defaults,
+            ...overrides,
+            evaluationParameters: {
+                ...defaults.evaluationParameters,
+                ...overrides.evaluationParameters,
+            },
+        };
+    };
+    describe('basic functionality', () => {
+        it('should return a valid EvaluationResult structure', () => {
+            const request = createRequest({
+                actualResponse: 'AI stands for artificial intelligence',
+                expectedKeywords: ['artificial intelligence'],
+            });
+            const result = performRougeLEvaluation(request);
+            expect(result).toMatchObject({
+                testCaseId: 'test-001',
+                passed: expect.any(Boolean),
+                keywordMatches: expect.any(Array),
+                sourceLinkMatches: expect.any(Array),
+                timestamp: expect.any(String),
+                evaluationParameters: expect.any(Object),
+                evaluationApproachResult: expect.any(Object),
+            });
+        });
+        it('should use default threshold when not provided', () => {
+            const request = createRequest({
+                evaluationParameters: { approach: 'rouge' },
+            });
+            const result = performRougeLEvaluation(request);
+            expect(result.evaluationParameters.threshold).toBe(DEFAULT_ROUGE_PASS_SCORE);
+        });
+        it('should use provided threshold when specified', () => {
+            const customThreshold = 0.85;
+            const request = createRequest({
+                actualResponse: 'response',
+                evaluationParameters: {
+                    approach: 'rouge',
+                    threshold: customThreshold,
+                },
+            });
+            const result = performRougeLEvaluation(request);
+            expect(result.evaluationParameters.threshold).toBe(customThreshold);
+        });
+    });
+    describe('single keyword evaluation', () => {
+        it('should pass when single-word keyword is found in candidate', () => {
+            const request = createRequest({
+                expectedKeywords: ['machine'],
+                actualResponse: 'This is about machine learning',
+            });
+            const result = performRougeLEvaluation(request);
+            expect(result).toMatchObject({
+                passed: true,
+                keywordMatches: [
+                    {
+                        keyword: 'machine',
+                        found: true,
+                        evaluationApproachResult: {
+                            score: 1,
+                            approachUsed: ROUGE,
+                        },
+                    },
+                ],
+            });
+        });
+        it('should fail when single-word keyword is not found in candidate', () => {
+            const request = createRequest({
+                expectedKeywords: ['quantum'],
+                actualResponse: 'This is about machine learning',
+            });
+            const result = performRougeLEvaluation(request);
+            expect(result).toMatchObject({
+                passed: false,
+                keywordMatches: [
+                    {
+                        found: false,
+                        evaluationApproachResult: {
+                            score: 0,
+                        },
+                    },
+                ],
+            });
+        });
+        it('should calculate ROUGE-L score for multi-word keywords', () => {
+            const request = createRequest({
+                expectedKeywords: ['machine learning'],
+                actualResponse: 'AI and machine learning are related',
+                evaluationParameters: { approach: 'rouge', threshold: 0.5 },
+            });
+            const result = performRougeLEvaluation(request);
+            expect(result).toMatchObject({
+                keywordMatches: [
+                    {
+                        found: true,
+                        evaluationApproachResult: {
+                            score: expect.closeTo(0.5),
+                            approachUsed: ROUGE,
+                        },
+                    },
+                ],
+            });
+        });
+        it('should handle LCS result as object with length property', () => {
+            const request = createRequest({
+                expectedKeywords: ['deep learning'],
+                actualResponse: 'Deep learning is a subset of machine learning',
+            });
+            const result = performRougeLEvaluation(request);
+            expect(result.keywordMatches[0].evaluationApproachResult.score).toBeGreaterThan(0);
+        });
+    });
+    describe('multiple keywords evaluation', () => {
+        it('should pass when all keywords meet threshold', () => {
+            const request = createRequest({
+                expectedKeywords: ['machine', 'learning', 'AI'],
+                actualResponse: 'Machine learning and AI are transformative technologies',
+            });
+            const result = performRougeLEvaluation(request);
+            expect(result).toMatchObject({
+                passed: true,
+                evaluationApproachResult: {
+                    score: 1,
+                    approachUsed: ROUGE,
+                },
+            });
+            expect(result.keywordMatches).toHaveLength(3);
+            expect(result.keywordMatches.every(match => match.found)).toBe(true);
+        });
+        it('should fail when not all keywords meet threshold', () => {
+            const request = createRequest({
+                expectedKeywords: ['machine', 'quantum', 'AI'],
+                actualResponse: 'Machine learning and AI are transformative',
+            });
+            const result = performRougeLEvaluation(request);
+            expect(result.passed).toBe(false);
+            expect(result.keywordMatches).toHaveLength(3);
+            expect(result.keywordMatches.filter(match => match.found)).toHaveLength(2);
+            expect(result.evaluationApproachResult.score).toBeCloseTo(2 / 3);
+        });
+        it('should calculate overall score as ratio of passed keywords', () => {
+            const request = createRequest({
+                expectedKeywords: ['alpha', 'beta', 'gamma', 'delta'],
+                actualResponse: 'alpha and beta are here',
+            });
+            const result = performRougeLEvaluation(request);
+            expect(result).toMatchObject({
+                passed: false,
+                evaluationApproachResult: {
+                    score: 0.5, // 2 out of 4
+                },
+            });
+        });
+    });
+    describe('edge cases', () => {
+        it('should handle empty keywords array', () => {
+            const request = createRequest({
+                expectedKeywords: [],
+                actualResponse: 'Some response',
+            });
+            const result = performRougeLEvaluation(request);
+            expect(result).toMatchObject({
+                passed: true,
+                keywordMatches: [],
+                evaluationApproachResult: {
+                    score: 1,
+                },
+            });
+        });
+        it('should handle empty actual response', () => {
+            const request = createRequest({
+                expectedKeywords: ['machine'],
+                actualResponse: '',
+            });
+            // Suppress expected warning
+            const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => { });
+            const result = performRougeLEvaluation(request);
+            expect(result).toMatchObject({
+                passed: false,
+                keywordMatches: [
+                    {
+                        found: false,
+                        evaluationApproachResult: {
+                            score: 0,
+                        },
+                    },
+                ],
+            });
+            consoleWarnSpy.mockRestore();
+        });
+        it('should handle whitespace-only keyword', () => {
+            const request = createRequest({
+                expectedKeywords: ['   '],
+                actualResponse: 'Some response',
+            });
+            // Suppress expected warning
+            const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => { });
+            const result = performRougeLEvaluation(request);
+            expect(result.keywordMatches[0]).toMatchObject({
+                found: false,
+                evaluationApproachResult: {
+                    score: 0,
+                },
+            });
+            consoleWarnSpy.mockRestore();
+        });
+        it('should handle null/undefined actualResponse gracefully', () => {
+            const request = createRequest({
+                expectedKeywords: ['machine'],
+                actualResponse: null,
+            });
+            // Suppress expected warning
+            const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => { });
+            const result = performRougeLEvaluation(request);
+            expect(result).toMatchObject({
+                passed: false,
+                keywordMatches: [
+                    {
+                        found: false,
+                    },
+                ],
+            });
+            consoleWarnSpy.mockRestore();
+        });
+    });
+    describe('case insensitivity', () => {
+        it('should perform case-insensitive matching', () => {
+            const request = createRequest({
+                expectedKeywords: ['MACHINE'],
+                actualResponse: 'machine learning is important',
+            });
+            const result = performRougeLEvaluation(request);
+            expect(result.keywordMatches[0]).toMatchObject({
+                found: true,
+                evaluationApproachResult: {
+                    score: 1,
+                },
+            });
+        });
+        it('should match keywords with mixed case', () => {
+            const request = createRequest({
+                expectedKeywords: ['MaChInE LeArNiNg'],
+                actualResponse: 'MACHINE LEARNING is a field of AI',
+                evaluationParameters: { approach: 'rouge', threshold: 0.4 }, // Lower threshold for real ROUGE-L behavior
+            });
+            const result = performRougeLEvaluation(request);
+            expect(result.keywordMatches[0]).toMatchObject({
+                found: true,
+            });
+            expect(result.keywordMatches[0].evaluationApproachResult.score).toBeGreaterThanOrEqual(0.4);
+        });
+    });
+    describe('ROUGE-L score calculation', () => {
+        it('should calculate correct F-score from precision and recall', () => {
+            const request = createRequest({
+                expectedKeywords: ['neural network'],
+                actualResponse: 'A neural network processes data',
+            });
+            const result = performRougeLEvaluation(request);
+            // With actual ROUGE-L: both words 'neural' and 'network' are found
+            // LCS length = 2, reference length = 2, candidate length = 5
+            // recall = 2/2 = 1.0, precision = 2/5 = 0.4
+            // F-score = 2 * (1.0 * 0.4) / (1.0 + 0.4) ≈ 0.571
+            const expectedFScore = (2 * 1.0 * 0.4) / (1.0 + 0.4);
+            expect(result.keywordMatches[0].evaluationApproachResult.score).toBeCloseTo(expectedFScore, 2);
+        });
+        it('should handle partial matches', () => {
+            const request = createRequest({
+                expectedKeywords: ['artificial intelligence systems'],
+                actualResponse: 'Artificial intelligence is growing',
+                evaluationParameters: { approach: 'rouge', threshold: 0.5 },
+            });
+            const result = performRougeLEvaluation(request);
+            // With actual ROUGE-L: 'artificial' and 'intelligence' are found, 'systems' is not
+            // LCS length = 2, reference length = 3, candidate length = 4
+            // recall = 2/3, precision = 2/4 = 0.5
+            const recall = 2 / 3;
+            const precision = 2 / 4;
+            const expectedFScore = (2 * precision * recall) / (precision + recall);
+            expect(result.keywordMatches[0].evaluationApproachResult.score).toBeCloseTo(expectedFScore, 2);
+        });
+    });
+    describe('timestamp', () => {
+        it('should include a valid ISO timestamp', () => {
+            const request = createRequest({
+                expectedKeywords: ['test'],
+                actualResponse: 'test response',
+            });
+            const result = performRougeLEvaluation(request);
+            expect(result.timestamp).toBeDefined();
+            expect(new Date(result.timestamp).toISOString()).toBe(result.timestamp);
+        });
+    });
+});
+//# sourceMappingURL=rougeL-evaluator.test.js.map

package/dist/collection/lib/evaluation/evaluators/rougeL-evaluator.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rougeL-evaluator.test.js","sourceRoot":"","sources":["../../../../src/lib/evaluation/evaluators/rougeL-evaluator.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAC3D,OAAO,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAE7D,OAAO,EAAE,wBAAwB,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAE9D,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,mEAAmE;IACnE,MAAM,aAAa,GAAG,CAAC,YAAwC,EAAE,EAAqB,EAAE;QACtF,MAAM,QAAQ,GAAsB;YAClC,UAAU,EAAE,UAAU;YACtB,QAAQ,EAAE,eAAe;YACzB,gBAAgB,EAAE,CAAC,SAAS,CAAC;YAC7B,mBAAmB,EAAE,EAAE;YACvB,cAAc,EAAE,uBAAuB;YACvC,oBAAoB,EAAE;gBACpB,QAAQ,EAAE,OAAO;gBACjB,SAAS,EAAE,wBAAwB;aACpC;SACF,CAAC;QAEF,OAAO;YACL,GAAG,QAAQ;YACX,GAAG,SAAS;YACZ,oBAAoB,EAAE;gBACpB,GAAG,QAAQ,CAAC,oBAAoB;gBAChC,GAAG,SAAS,CAAC,oBAAoB;aAClC;SACF,CAAC;IACJ,CAAC,CAAC;IAEF,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,cAAc,EAAE,uCAAuC;gBACvD,gBAAgB,EAAE,CAAC,yBAAyB,CAAC;aAC9C,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEhD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC;gBAC3B,UAAU,EAAE,UAAU;gBACtB,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC;gBAC3B,cAAc,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;gBACjC,iBAAiB,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;gBACpC,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;gBAC7B,oBAAoB,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;gBACxC,wBAAwB,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;aAC7C,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;YACxD,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,oBAAoB,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE;aAC5C,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEhD,MAAM,CAAC,MAAM,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;YACtD,MAAM,eAAe,GAAG,IAAI,CAAC;YAC7B,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,cAAc,EAAE,UAAU;gBAC1B,oBAAoB,EAAE;oBACpB,QAAQ,EAAE,OAAO;oBACjB,SAAS,EAAE,eAAe;iBAC3B;aACF,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEhD,MAAM,CAAC,MAAM,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACzC,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;YACpE,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,gBAAgB,EAAE,CAAC,SAAS,CAAC;gBAC7B,cAAc,EAAE,gCAAgC;aACjD,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEhD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC;gBAC3B,MAAM,EAAE,IAAI;gBACZ,cAAc,EAAE;oBACd;wBACE,OAAO,EAAE,SAAS;wBAClB,KAAK,EAAE,IAAI;wBACX,wBAAwB,EAAE;4BACxB,KAAK,EAAE,CAAC;4BACR,YAAY,EAAE,KAAK;yBACpB;qBACF;iBACF;aACF,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;YACxE,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,gBAAgB,EAAE,CAAC,SAAS,CAAC;gBAC7B,cAAc,EAAE,gCAAgC;aACjD,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEhD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC;gBAC3B,MAAM,EAAE,KAAK;gBACb,cAAc,EAAE;oBACd;wBACE,KAAK,EAAE,KAAK;wBACZ,wBAAwB,EAAE;4BACxB,KAAK,EAAE,CAAC;yBACT;qBACF;iBACF;aACF,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;YAChE,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,gBAAgB,EAAE,CAAC,kBAAkB,CAAC;gBACtC,cAAc,EAAE,qCAAqC;gBACrD,oBAAoB,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE;aAC5D,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEhD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC;gBAC3B,cAAc,EAAE;oBACd;wBACE,KAAK,EAAE,IAAI;wBACX,wBAAwB,EAAE;4BACxB,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC;4BAC1B,YAAY,EAAE,KAAK;yBACpB;qBACF;iBACF;aACF,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;YACjE,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,gBAAgB,EAAE,CAAC,eAAe,CAAC;gBACnC,cAAc,EAAE,+CAA+C;aAChE,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEhD,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACrF,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,8BAA8B,EAAE,GAAG,EAAE;QAC5C,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;YACtD,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,gBAAgB,EAAE,CAAC,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC;gBAC/C,cAAc,EAAE,yDAAyD;aAC1E,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEhD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC;gBAC3B,MAAM,EAAE,IAAI;gBACZ,wBAAwB,EAAE;oBACxB,KAAK,EAAE,CAAC;oBACR,YAAY,EAAE,KAAK;iBACpB;aACF,CAAC,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,gBAAgB,EAAE,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC;gBAC9C,cAAc,EAAE,4CAA4C;aAC7D,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEhD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC3E,MAAM,CAAC,MAAM,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;YACpE,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,gBAAgB,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC;gBACrD,cAAc,EAAE,yBAAyB;aAC1C,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEhD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC;gBAC3B,MAAM,EAAE,KAAK;gBACb,wBAAwB,EAAE;oBACxB,KAAK,EAAE,GAAG,EAAE,aAAa;iBAC1B;aACF,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;QAC1B,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,gBAAgB,EAAE,EAAE;gBACpB,cAAc,EAAE,eAAe;aAChC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEhD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC;gBAC3B,MAAM,EAAE,IAAI;gBACZ,cAAc,EAAE,EAAE;gBAClB,wBAAwB,EAAE;oBACxB,KAAK,EAAE,CAAC;iBACT;aACF,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,gBAAgB,EAAE,CAAC,SAAS,CAAC;gBAC7B,cAAc,EAAE,EAAE;aACnB,CAAC,CAAC;YAEH,4BAA4B;YAC5B,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YAEhF,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEhD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC;gBAC3B,MAAM,EAAE,KAAK;gBACb,cAAc,EAAE;oBACd;wBACE,KAAK,EAAE,KAAK;wBACZ,wBAAwB,EAAE;4BACxB,KAAK,EAAE,CAAC;yBACT;qBACF;iBACF;aACF,CAAC,CAAC;YAEH,cAAc,CAAC,WAAW,EAAE,CAAC;QAC/B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,gBAAgB,EAAE,CAAC,KAAK,CAAC;gBACzB,cAAc,EAAE,eAAe;aAChC,CAAC,CAAC;YAEH,4BAA4B;YAC5B,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YAEhF,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEhD,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;gBAC7C,KAAK,EAAE,KAAK;gBACZ,wBAAwB,EAAE;oBACxB,KAAK,EAAE,CAAC;iBACT;aACF,CAAC,CAAC;YAEH,cAAc,CAAC,WAAW,EAAE,CAAC;QAC/B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;YAChE,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,gBAAgB,EAAE,CAAC,SAAS,CAAC;gBAC7B,cAAc,EAAE,IAAyB;aAC1C,CAAC,CAAC;YAEH,4BAA4B;YAC5B,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YAEhF,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEhD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC;gBAC3B,MAAM,EAAE,KAAK;gBACb,cAAc,EAAE;oBACd;wBACE,KAAK,EAAE,KAAK;qBACb;iBACF;aACF,CAAC,CAAC;YAEH,cAAc,CAAC,WAAW,EAAE,CAAC;QAC/B,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,gBAAgB,EAAE,CAAC,SAAS,CAAC;gBAC7B,cAAc,EAAE,+BAA+B;aAChD,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEhD,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;gBAC7C,KAAK,EAAE,IAAI;gBACX,wBAAwB,EAAE;oBACxB,KAAK,EAAE,CAAC;iBACT;aACF,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,gBAAgB,EAAE,CAAC,kBAAkB,CAAC;gBACtC,cAAc,EAAE,mCAAmC;gBACnD,oBAAoB,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,EAAE,4CAA4C;aAC1G,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEhD,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;gBAC7C,KAAK,EAAE,IAAI;aACZ,CAAC,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;QAC9F,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACzC,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;YACpE,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,gBAAgB,EAAE,CAAC,gBAAgB,CAAC;gBACpC,cAAc,EAAE,iCAAiC;aAClD,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEhD,mEAAmE;YACnE,6DAA6D;YAC7D,4CAA4C;YAC5C,kDAAkD;YAClD,MAAM,cAAc,GAAG,CAAC,CAAC,GAAG,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;YACrD,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QACjG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,gBAAgB,EAAE,CAAC,iCAAiC,CAAC;gBACrD,cAAc,EAAE,oCAAoC;gBACpD,oBAAoB,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE;aAC5D,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEhD,mFAAmF;YACnF,6DAA6D;YAC7D,sCAAsC;YACtC,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC;YACrB,MAAM,SAAS,GAAG,CAAC,GAAG,CAAC,CAAC;YACxB,MAAM,cAAc,GAAG,CAAC,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC;YAEvE,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QACjG,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;QACzB,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;YAC9C,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,gBAAgB,EAAE,CAAC,MAAM,CAAC;gBAC1B,cAAc,EAAE,eAAe;aAChC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAEhD,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAU,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC3E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC","sourcesContent":["import { jest, describe, it, expect } from '@jest/globals';\nimport { performRougeLEvaluation } from './rougeL-evaluator';\nimport { EvaluationRequest } from '../types';\nimport { DEFAULT_ROUGE_PASS_SCORE, ROUGE } from '../constant';\n\ndescribe('performRougeLEvaluation', () => {\n  // Helper function to create a base request with optional overrides\n  const createRequest = (overrides: Partial<EvaluationRequest> = {}): EvaluationRequest => {\n    const defaults: EvaluationRequest = {\n      testCaseId: 'test-001',\n      question: 'Test question',\n      expectedKeywords: ['keyword'],\n      expectedSourceLinks: [],\n      actualResponse: 'response with keyword',\n      evaluationParameters: {\n        approach: 'rouge',\n        threshold: DEFAULT_ROUGE_PASS_SCORE,\n      },\n    };\n\n    return {\n      ...defaults,\n      ...overrides,\n      evaluationParameters: {\n        ...defaults.evaluationParameters,\n        ...overrides.evaluationParameters,\n      },\n    };\n  };\n\n  describe('basic functionality', () => {\n    it('should return a valid EvaluationResult structure', () => {\n      const request = createRequest({\n        actualResponse: 'AI stands for artificial intelligence',\n        expectedKeywords: ['artificial intelligence'],\n      });\n\n      const result = performRougeLEvaluation(request);\n\n      expect(result).toMatchObject({\n        testCaseId: 'test-001',\n        passed: expect.any(Boolean),\n        keywordMatches: expect.any(Array),\n        sourceLinkMatches: expect.any(Array),\n        timestamp: expect.any(String),\n        evaluationParameters: expect.any(Object),\n        evaluationApproachResult: expect.any(Object),\n      });\n    });\n\n    it('should use default threshold when not provided', () => {\n      const request = createRequest({\n        evaluationParameters: { approach: 'rouge' },\n      });\n\n      const result = performRougeLEvaluation(request);\n\n      expect(result.evaluationParameters.threshold).toBe(DEFAULT_ROUGE_PASS_SCORE);\n    });\n\n    it('should use provided threshold when specified', () => {\n      const customThreshold = 0.85;\n      const request = createRequest({\n        actualResponse: 'response',\n        evaluationParameters: {\n          approach: 'rouge',\n          threshold: customThreshold,\n        },\n      });\n\n      const result = performRougeLEvaluation(request);\n\n      expect(result.evaluationParameters.threshold).toBe(customThreshold);\n    });\n  });\n\n  describe('single keyword evaluation', () => {\n    it('should pass when single-word keyword is found in candidate', () => {\n      const request = createRequest({\n        expectedKeywords: ['machine'],\n        actualResponse: 'This is about machine learning',\n      });\n\n      const result = performRougeLEvaluation(request);\n\n      expect(result).toMatchObject({\n        passed: true,\n        keywordMatches: [\n          {\n            keyword: 'machine',\n            found: true,\n            evaluationApproachResult: {\n              score: 1,\n              approachUsed: ROUGE,\n            },\n          },\n        ],\n      });\n    });\n\n    it('should fail when single-word keyword is not found in candidate', () => {\n      const request = createRequest({\n        expectedKeywords: ['quantum'],\n        actualResponse: 'This is about machine learning',\n      });\n\n      const result = performRougeLEvaluation(request);\n\n      expect(result).toMatchObject({\n        passed: false,\n        keywordMatches: [\n          {\n            found: false,\n            evaluationApproachResult: {\n              score: 0,\n            },\n          },\n        ],\n      });\n    });\n\n    it('should calculate ROUGE-L score for multi-word keywords', () => {\n      const request = createRequest({\n        expectedKeywords: ['machine learning'],\n        actualResponse: 'AI and machine learning are related',\n        evaluationParameters: { approach: 'rouge', threshold: 0.5 },\n      });\n\n      const result = performRougeLEvaluation(request);\n\n      expect(result).toMatchObject({\n        keywordMatches: [\n          {\n            found: true,\n            evaluationApproachResult: {\n              score: expect.closeTo(0.5),\n              approachUsed: ROUGE,\n            },\n          },\n        ],\n      });\n    });\n\n    it('should handle LCS result as object with length property', () => {\n      const request = createRequest({\n        expectedKeywords: ['deep learning'],\n        actualResponse: 'Deep learning is a subset of machine learning',\n      });\n\n      const result = performRougeLEvaluation(request);\n\n      expect(result.keywordMatches[0].evaluationApproachResult.score).toBeGreaterThan(0);\n    });\n  });\n\n  describe('multiple keywords evaluation', () => {\n    it('should pass when all keywords meet threshold', () => {\n      const request = createRequest({\n        expectedKeywords: ['machine', 'learning', 'AI'],\n        actualResponse: 'Machine learning and AI are transformative technologies',\n      });\n\n      const result = performRougeLEvaluation(request);\n\n      expect(result).toMatchObject({\n        passed: true,\n        evaluationApproachResult: {\n          score: 1,\n          approachUsed: ROUGE,\n        },\n      });\n      expect(result.keywordMatches).toHaveLength(3);\n      expect(result.keywordMatches.every(match => match.found)).toBe(true);\n    });\n\n    it('should fail when not all keywords meet threshold', () => {\n      const request = createRequest({\n        expectedKeywords: ['machine', 'quantum', 'AI'],\n        actualResponse: 'Machine learning and AI are transformative',\n      });\n\n      const result = performRougeLEvaluation(request);\n\n      expect(result.passed).toBe(false);\n      expect(result.keywordMatches).toHaveLength(3);\n      expect(result.keywordMatches.filter(match => match.found)).toHaveLength(2);\n      expect(result.evaluationApproachResult.score).toBeCloseTo(2 / 3);\n    });\n\n    it('should calculate overall score as ratio of passed keywords', () => {\n      const request = createRequest({\n        expectedKeywords: ['alpha', 'beta', 'gamma', 'delta'],\n        actualResponse: 'alpha and beta are here',\n      });\n\n      const result = performRougeLEvaluation(request);\n\n      expect(result).toMatchObject({\n        passed: false,\n        evaluationApproachResult: {\n          score: 0.5, // 2 out of 4\n        },\n      });\n    });\n  });\n\n  describe('edge cases', () => {\n    it('should handle empty keywords array', () => {\n      const request = createRequest({\n        expectedKeywords: [],\n        actualResponse: 'Some response',\n      });\n\n      const result = performRougeLEvaluation(request);\n\n      expect(result).toMatchObject({\n        passed: true,\n        keywordMatches: [],\n        evaluationApproachResult: {\n          score: 1,\n        },\n      });\n    });\n\n    it('should handle empty actual response', () => {\n      const request = createRequest({\n        expectedKeywords: ['machine'],\n        actualResponse: '',\n      });\n\n      // Suppress expected warning\n      const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});\n\n      const result = performRougeLEvaluation(request);\n\n      expect(result).toMatchObject({\n        passed: false,\n        keywordMatches: [\n          {\n            found: false,\n            evaluationApproachResult: {\n              score: 0,\n            },\n          },\n        ],\n      });\n\n      consoleWarnSpy.mockRestore();\n    });\n\n    it('should handle whitespace-only keyword', () => {\n      const request = createRequest({\n        expectedKeywords: ['   '],\n        actualResponse: 'Some response',\n      });\n\n      // Suppress expected warning\n      const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});\n\n      const result = performRougeLEvaluation(request);\n\n      expect(result.keywordMatches[0]).toMatchObject({\n        found: false,\n        evaluationApproachResult: {\n          score: 0,\n        },\n      });\n\n      consoleWarnSpy.mockRestore();\n    });\n\n    it('should handle null/undefined actualResponse gracefully', () => {\n      const request = createRequest({\n        expectedKeywords: ['machine'],\n        actualResponse: null as unknown as string,\n      });\n\n      // Suppress expected warning\n      const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});\n\n      const result = performRougeLEvaluation(request);\n\n      expect(result).toMatchObject({\n        passed: false,\n        keywordMatches: [\n          {\n            found: false,\n          },\n        ],\n      });\n\n      consoleWarnSpy.mockRestore();\n    });\n  });\n\n  describe('case insensitivity', () => {\n    it('should perform case-insensitive matching', () => {\n      const request = createRequest({\n        expectedKeywords: ['MACHINE'],\n        actualResponse: 'machine learning is important',\n      });\n\n      const result = performRougeLEvaluation(request);\n\n      expect(result.keywordMatches[0]).toMatchObject({\n        found: true,\n        evaluationApproachResult: {\n          score: 1,\n        },\n      });\n    });\n\n    it('should match keywords with mixed case', () => {\n      const request = createRequest({\n        expectedKeywords: ['MaChInE LeArNiNg'],\n        actualResponse: 'MACHINE LEARNING is a field of AI',\n        evaluationParameters: { approach: 'rouge', threshold: 0.4 }, // Lower threshold for real ROUGE-L behavior\n      });\n\n      const result = performRougeLEvaluation(request);\n\n      expect(result.keywordMatches[0]).toMatchObject({\n        found: true,\n      });\n      expect(result.keywordMatches[0].evaluationApproachResult.score).toBeGreaterThanOrEqual(0.4);\n    });\n  });\n\n  describe('ROUGE-L score calculation', () => {\n    it('should calculate correct F-score from precision and recall', () => {\n      const request = createRequest({\n        expectedKeywords: ['neural network'],\n        actualResponse: 'A neural network processes data',\n      });\n\n      const result = performRougeLEvaluation(request);\n\n      // With actual ROUGE-L: both words 'neural' and 'network' are found\n      // LCS length = 2, reference length = 2, candidate length = 5\n      // recall = 2/2 = 1.0, precision = 2/5 = 0.4\n      // F-score = 2 * (1.0 * 0.4) / (1.0 + 0.4) ≈ 0.571\n      const expectedFScore = (2 * 1.0 * 0.4) / (1.0 + 0.4);\n      expect(result.keywordMatches[0].evaluationApproachResult.score).toBeCloseTo(expectedFScore, 2);\n    });\n\n    it('should handle partial matches', () => {\n      const request = createRequest({\n        expectedKeywords: ['artificial intelligence systems'],\n        actualResponse: 'Artificial intelligence is growing',\n        evaluationParameters: { approach: 'rouge', threshold: 0.5 },\n      });\n\n      const result = performRougeLEvaluation(request);\n\n      // With actual ROUGE-L: 'artificial' and 'intelligence' are found, 'systems' is not\n      // LCS length = 2, reference length = 3, candidate length = 4\n      // recall = 2/3, precision = 2/4 = 0.5\n      const recall = 2 / 3;\n      const precision = 2 / 4;\n      const expectedFScore = (2 * precision * recall) / (precision + recall);\n\n      expect(result.keywordMatches[0].evaluationApproachResult.score).toBeCloseTo(expectedFScore, 2);\n    });\n  });\n\n  describe('timestamp', () => {\n    it('should include a valid ISO timestamp', () => {\n      const request = createRequest({\n        expectedKeywords: ['test'],\n        actualResponse: 'test response',\n      });\n\n      const result = performRougeLEvaluation(request);\n\n      expect(result.timestamp).toBeDefined();\n      expect(new Date(result.timestamp!).toISOString()).toBe(result.timestamp);\n    });\n  });\n});\n"]}

package/dist/collection/lib/evaluation/evaluators/semantic/SemanticEvaluator.js

@@ -0,0 +1,63 @@
+import { loadSemanticModel } from "./model-loader";
+import { evaluateKeywordsSemantically } from "./evaluate-keywords";
+import { evaluateSourceLinks } from "../../evaluators/exact/exact";
+import { DEFAULT_SEMANTIC_PASS_SCORE } from "../../constant";
+import { EvaluationApproach } from "../../constants/evaluation-approach";
+export class SemanticEvaluator {
+    // TODO(LLM-39): Refactor SemanticEvaluator into a singleton pattern.
+    static extractor = null;
+    async initialize() {
+        if (SemanticEvaluator.extractor)
+            return;
+        try {
+            SemanticEvaluator.extractor = await loadSemanticModel();
+        }
+        catch (error) {
+            console.error('Failed to load semantic evaluation model:', error);
+            throw error;
+        }
+    }
+    async performEvaluation(request) {
+        try {
+            await this.initialize();
+            const keywordMatches = await evaluateKeywordsSemantically(SemanticEvaluator.extractor, request.actualResponse, request.expectedKeywords, DEFAULT_SEMANTIC_PASS_SCORE);
+            const sourceLinkMatches = evaluateSourceLinks(request.expectedSourceLinks, request.actualResponse);
+            const totalItems = keywordMatches.length + sourceLinkMatches.length;
+            // calculate the overall score by averaging the score of the keyword and source link matches
+            const keywordScore = keywordMatches.reduce((acc, curr) => acc + curr.evaluationApproachResult.score, 0);
+            const sourceLinkScore = sourceLinkMatches.reduce((acc, curr) => acc + curr.evaluationApproachResult.score, 0);
+            const totalScore = keywordScore + sourceLinkScore;
+            const overallScore = totalItems > 0 ? totalScore / totalItems : 0; // to avoid division by zero
+            const passed = keywordMatches.every(match => match.found) && sourceLinkMatches.every(match => match.found);
+            const evaluationParameters = {
+                approach: EvaluationApproach.SEMANTIC,
+                threshold: DEFAULT_SEMANTIC_PASS_SCORE,
+            };
+            return {
+                testCaseId: request.testCaseId,
+                passed,
+                keywordMatches,
+                sourceLinkMatches,
+                evaluationParameters,
+                evaluationApproachResult: { score: overallScore, approachUsed: EvaluationApproach.SEMANTIC },
+                timestamp: new Date().toISOString(),
+            };
+        }
+        catch (error) {
+            console.error('Failed to perform semantic evaluation:', error);
+            return {
+                testCaseId: request.testCaseId,
+                passed: false,
+                keywordMatches: [],
+                sourceLinkMatches: [],
+                evaluationParameters: {
+                    approach: EvaluationApproach.SEMANTIC,
+                    threshold: DEFAULT_SEMANTIC_PASS_SCORE,
+                },
+                evaluationApproachResult: { score: 0, approachUsed: EvaluationApproach.SEMANTIC },
+                timestamp: new Date().toISOString(),
+            };
+        }
+    }
+}
+//# sourceMappingURL=SemanticEvaluator.js.map

package/dist/collection/lib/evaluation/evaluators/semantic/SemanticEvaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SemanticEvaluator.js","sourceRoot":"","sources":["../../../../../src/lib/evaluation/evaluators/semantic/SemanticEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAE,4BAA4B,EAAE,MAAM,qBAAqB,CAAC;AAEnE,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AAEnE,OAAO,EAAE,2BAA2B,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AAEzE,MAAM,OAAO,iBAAiB;IAC5B,qEAAqE;IAC7D,MAAM,CAAC,SAAS,GAA8B,IAAI,CAAC;IAE3D,KAAK,CAAC,UAAU;QACd,IAAI,iBAAiB,CAAC,SAAS;YAAE,OAAO;QACxC,IAAI,CAAC;YACH,iBAAiB,CAAC,SAAS,GAAG,MAAM,iBAAiB,EAAE,CAAC;QAC1D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,2CAA2C,EAAE,KAAK,CAAC,CAAC;YAClE,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,OAA0B;QAChD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;YACxB,MAAM,cAAc,GAAG,MAAM,4BAA4B,CAAC,iBAAiB,CAAC,SAAS,EAAE,OAAO,CAAC,cAAc,EAAE,OAAO,CAAC,gBAAgB,EAAE,2BAA2B,CAAC,CAAC;YACtK,MAAM,iBAAiB,GAAG,mBAAmB,CAAC,OAAO,CAAC,mBAAmB,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC;YACnG,MAAM,UAAU,GAAG,cAAc,CAAC,MAAM,GAAG,iBAAiB,CAAC,MAAM,CAAC;YACpE,4FAA4F;YAC5F,MAAM,YAAY,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,wBAAwB,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YACxG,MAAM,eAAe,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,wBAAwB,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YAC9G,MAAM,UAAU,GAAG,YAAY,GAAG,eAAe,CAAC;YAClD,MAAM,YAAY,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,4BAA4B;YAC/F,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,iBAAiB,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAE3G,MAAM,oBAAoB,GAAG;gBAC3B,QAAQ,EAAE,kBAAkB,CAAC,QAAQ;gBACrC,SAAS,EAAE,2BAA2B;aACf,CAAC;YAE1B,OAAO;gBACL,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,MAAM;gBACN,cAAc;gBACd,iBAAiB;gBACjB,oBAAoB;gBACpB,wBAAwB,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,kBAAkB,CAAC,QAAQ,EAAE;gBAC5F,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,wCAAwC,EAAE,KAAK,CAAC,CAAC;YAC/D,OAAO;gBACL,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,MAAM,EAAE,KAAK;gBACb,cAAc,EAAE,EAAE;gBAClB,iBAAiB,EAAE,EAAE;gBACrB,oBAAoB,EAAE;oBACpB,QAAQ,EAAE,kBAAkB,CAAC,QAAQ;oBACrC,SAAS,EAAE,2BAA2B;iBACvC;gBACD,wBAAwB,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,YAAY,EAAE,kBAAkB,CAAC,QAAQ,EAAE;gBACjF,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC;QACJ,CAAC;IACH,CAAC","sourcesContent":["import { EvaluationResult, EvaluationRequest } from '../../types';\nimport { loadSemanticModel } from './model-loader';\nimport { evaluateKeywordsSemantically } from './evaluate-keywords';\nimport { FeatureExtractionPipeline } from '@xenova/transformers';\nimport { evaluateSourceLinks } from '../../evaluators/exact/exact';\nimport { EvaluationParameters } from '../../../../types/evaluation';\nimport { DEFAULT_SEMANTIC_PASS_SCORE } from '../../constant';\nimport { EvaluationApproach } from '../../constants/evaluation-approach';\n\nexport class SemanticEvaluator {\n  // TODO(LLM-39): Refactor SemanticEvaluator into a singleton pattern.\n  private static extractor: FeatureExtractionPipeline = null;\n\n  async initialize(): Promise<void> {\n    if (SemanticEvaluator.extractor) return;\n    try {\n      SemanticEvaluator.extractor = await loadSemanticModel();\n    } catch (error) {\n      console.error('Failed to load semantic evaluation model:', error);\n      throw error;\n    }\n  }\n\n  async performEvaluation(request: EvaluationRequest): Promise<EvaluationResult> {\n    try {\n      await this.initialize();\n      const keywordMatches = await evaluateKeywordsSemantically(SemanticEvaluator.extractor, request.actualResponse, request.expectedKeywords, DEFAULT_SEMANTIC_PASS_SCORE);\n      const sourceLinkMatches = evaluateSourceLinks(request.expectedSourceLinks, request.actualResponse);\n      const totalItems = keywordMatches.length + sourceLinkMatches.length;\n      // calculate the overall score by averaging the score of the keyword and source link matches\n      const keywordScore = keywordMatches.reduce((acc, curr) => acc + curr.evaluationApproachResult.score, 0);\n      const sourceLinkScore = sourceLinkMatches.reduce((acc, curr) => acc + curr.evaluationApproachResult.score, 0);\n      const totalScore = keywordScore + sourceLinkScore;\n      const overallScore = totalItems > 0 ? totalScore / totalItems : 0; // to avoid division by zero\n      const passed = keywordMatches.every(match => match.found) && sourceLinkMatches.every(match => match.found);\n\n      const evaluationParameters = {\n        approach: EvaluationApproach.SEMANTIC,\n        threshold: DEFAULT_SEMANTIC_PASS_SCORE,\n      } as EvaluationParameters;\n\n      return {\n        testCaseId: request.testCaseId,\n        passed,\n        keywordMatches,\n        sourceLinkMatches,\n        evaluationParameters,\n        evaluationApproachResult: { score: overallScore, approachUsed: EvaluationApproach.SEMANTIC },\n        timestamp: new Date().toISOString(),\n      };\n    } catch (error) {\n      console.error('Failed to perform semantic evaluation:', error);\n      return {\n        testCaseId: request.testCaseId,\n        passed: false,\n        keywordMatches: [],\n        sourceLinkMatches: [],\n        evaluationParameters: {\n          approach: EvaluationApproach.SEMANTIC,\n          threshold: DEFAULT_SEMANTIC_PASS_SCORE,\n        },\n        evaluationApproachResult: { score: 0, approachUsed: EvaluationApproach.SEMANTIC },\n        timestamp: new Date().toISOString(),\n      };\n    }\n  }\n}\n"]}