@juspay/neurolink 9.30.0 → 9.31.0

package/dist/client/workflow/core/judgeScorer.js

@@ -0,0 +1,544 @@
+/**
+ * workflow/core/judgeScorer.ts
+ * Judge-based scoring system for ensemble response evaluation
+ */
+import { AIProviderFactory } from "../../core/factory.js";
+import { logger } from "../../utils/logger.js";
+import { SpanSerializer, SpanType, SpanStatus, getMetricsAggregator, } from "../../observability/index.js";
+import { MAX_REASONING_LENGTH } from "../config.js";
+import { WorkflowError } from "../types.js";
+const functionTag = "JudgeScorer";
+// ============================================================================
+// SCORING FUNCTIONS
+// ============================================================================
+/**
+ * Execute judge scoring on ensemble responses
+ * @param options - Scoring options including judges and responses
+ * @returns Score result with judge evaluation
+ */
+export async function scoreEnsemble(options) {
+    const startTime = Date.now();
+    const { judges, responses, originalPrompt, systemPrompt, timeout, workflowDefaults, } = options;
+    const span = SpanSerializer.createSpan(SpanType.WORKFLOW, "workflow.judge", {
+        "workflow.operation": "judge",
+        "workflow.judge_count": judges.length,
+        "workflow.response_count": responses.length,
+        "workflow.pattern": judges.length > 1 ? "multi-judge" : "single-judge",
+    });
+    logger.info(`[${functionTag}] Starting judge scoring`, {
+        judgeCount: judges.length,
+        responseCount: responses.length,
+    });
+    try {
+        // Filter successful responses for evaluation
+        const successfulResponses = responses.filter((r) => r.status === "success" && r.content.trim() !== "");
+        if (successfulResponses.length === 0) {
+            throw new WorkflowError("No successful responses to evaluate", {
+                code: "NO_RESPONSES_TO_EVALUATE",
+                workflowId: "judge",
+                phase: "judge",
+                retryable: false,
+            });
+        }
+        if (judges.length === 1) {
+            // Single judge scoring
+            const judgeResult = await executeSingleJudge(judges[0], successfulResponses, originalPrompt, systemPrompt, timeout, workflowDefaults?.judgePrompt);
+            const judgeTime = Date.now() - startTime;
+            span.durationMs = judgeTime;
+            const endedSpan = SpanSerializer.endSpan(span, SpanStatus.OK);
+            getMetricsAggregator().recordSpan(endedSpan);
+            return {
+                scores: judgeResult,
+                judgeTime,
+            };
+        }
+        else {
+            // Multi-judge voting
+            const multiJudgeResult = await executeMultiJudge(judges, successfulResponses, originalPrompt, systemPrompt, timeout, workflowDefaults?.judgePrompt);
+            const judgeTime = Date.now() - startTime;
+            span.durationMs = judgeTime;
+            const endedSpan = SpanSerializer.endSpan(span, SpanStatus.OK);
+            getMetricsAggregator().recordSpan(endedSpan);
+            return {
+                scores: multiJudgeResult,
+                judgeTime,
+            };
+        }
+    }
+    catch (error) {
+        const err = error;
+        logger.error(`[${functionTag}] Judge scoring failed`, {
+            error: err.message,
+        });
+        span.durationMs = Date.now() - startTime;
+        const endedSpan = SpanSerializer.endSpan(span, SpanStatus.ERROR, err.message);
+        getMetricsAggregator().recordSpan(endedSpan);
+        const workflowError = error instanceof WorkflowError
+            ? error
+            : new WorkflowError(err.message, {
+                code: "JUDGE_SCORING_ERROR",
+                workflowId: "judge",
+                phase: "judge",
+                retryable: true,
+            });
+        return {
+            scores: createEmptyScores(judges[0], responses),
+            judgeTime: Date.now() - startTime,
+            error: workflowError,
+        };
+    }
+}
+/**
+ * Execute single judge evaluation
+ * @param judge - Judge configuration
+ * @param responses - Successful ensemble responses
+ * @param originalPrompt - Original user prompt
+ * @param systemPrompt - Optional system prompt
+ * @param timeout - Judge timeout in milliseconds
+ * @returns Judge scores with evaluation
+ */
+async function executeSingleJudge(judge, responses, originalPrompt, systemPrompt, timeout, workflowDefaultJudgePrompt) {
+    const startTime = Date.now();
+    logger.debug(`[${functionTag}] Executing single judge`, {
+        provider: judge.provider,
+        model: judge.model,
+    });
+    // Resolve judge prompt with hierarchical fallback:
+    // 1. Judge-specific customPrompt (highest priority)
+    // 2. Workflow-level default judge prompt
+    // 3. Built-in default template
+    const resolvedJudgePrompt = judge.customPrompt || workflowDefaultJudgePrompt;
+    // Create judge prompt (will use resolvedJudgePrompt if provided, otherwise default template)
+    const judgePrompt = createJudgePrompt(judge, responses, originalPrompt, resolvedJudgePrompt);
+    // Execute judge
+    const provider = await AIProviderFactory.createProvider(judge.provider, judge.model);
+    const result = await provider.generate({
+        prompt: judgePrompt,
+        systemPrompt: systemPrompt || judge.systemPrompt,
+        temperature: judge.temperature || 0.1,
+        maxTokens: judge.maxTokens || 2000,
+        timeout: timeout || judge.timeout || 10000,
+    });
+    // Parse judge response
+    const parsed = parseJudgeResponse(result?.content || "", responses, judge);
+    // Build JudgeScores
+    const judgeScores = {
+        judgeProvider: judge.provider,
+        judgeModel: judge.model,
+        scores: parsed.scores,
+        ranking: parsed.ranking,
+        bestResponse: parsed.bestResponse,
+        criteria: judge.criteria,
+        reasoning: parsed.reasoning,
+        synthesizedResponse: parsed.synthesizedResponse, // Include synthesized response if present
+        confidenceInJudgment: parsed.confidenceInJudgment,
+        judgeTime: Date.now() - startTime,
+        timestamp: new Date().toISOString(),
+    };
+    logger.debug(`[${functionTag}] Single judge completed`, {
+        bestResponse: judgeScores.bestResponse,
+        hasSynthesizedResponse: !!parsed.synthesizedResponse,
+        judgeTime: judgeScores.judgeTime,
+    });
+    return judgeScores;
+}
+/**
+ * Execute multi-judge voting
+ * @param judges - Array of judge configurations
+ * @param responses - Successful ensemble responses
+ * @param originalPrompt - Original user prompt
+ * @param systemPrompt - Optional system prompt
+ * @param timeout - Judge timeout in milliseconds
+ * @returns Multi-judge scores with aggregated results
+ */
+async function executeMultiJudge(judges, responses, originalPrompt, systemPrompt, timeout, workflowDefaultJudgePrompt) {
+    const startTime = Date.now();
+    logger.debug(`[${functionTag}] Executing multi-judge voting`, {
+        judgeCount: judges.length,
+    });
+    // Execute all judges in parallel
+    const judgePromises = judges.map((judge) => executeSingleJudge(judge, responses, originalPrompt, systemPrompt, timeout, workflowDefaultJudgePrompt).catch((error) => {
+        logger.warn(`[${functionTag}] Judge failed`, {
+            provider: judge.provider,
+            model: judge.model,
+            error: error.message,
+        });
+        return createEmptyJudgeScores(judge, responses);
+    }));
+    const judgeResults = await Promise.all(judgePromises);
+    // Aggregate scores using average (can be configurable in future)
+    const aggregated = aggregateJudgeScores(judgeResults, "average");
+    const multiJudgeScores = {
+        judges: judgeResults,
+        averageScores: aggregated.averageScores,
+        aggregatedRanking: aggregated.ranking,
+        consensusLevel: calculateConsensusLevel(judgeResults),
+        bestResponse: aggregated.bestResponse,
+        confidence: aggregated.confidence,
+        votingStrategy: "average",
+        // Expose unified interface fields
+        judgeProvider: judgeResults[0]?.judgeProvider,
+        judgeModel: `multi-judge-${judges.length}`,
+        scores: aggregated.averageScores,
+        ranking: aggregated.ranking,
+        reasoning: aggregated.reasoning,
+        confidenceInJudgment: aggregated.confidence,
+        criteria: judges[0]?.criteria || [],
+        judgeTime: Date.now() - startTime,
+        timestamp: new Date().toISOString(),
+    };
+    logger.debug(`[${functionTag}] Multi-judge completed`, {
+        bestResponse: multiJudgeScores.bestResponse,
+        consensusLevel: multiJudgeScores.consensusLevel,
+        judgeTime: multiJudgeScores.judgeTime,
+    });
+    return multiJudgeScores;
+}
+// ============================================================================
+// HELPER FUNCTIONS
+// ============================================================================
+/**
+ * Create judge evaluation prompt
+ * @param judge - Judge configuration
+ * @param responses - Ensemble responses to evaluate
+ * @param originalPrompt - Original user prompt
+ * @param customPrompt - Custom evaluation prompt (overrides default)
+ * @returns Formatted judge prompt
+ */
+function createJudgePrompt(judge, responses, originalPrompt, customPrompt) {
+    // If custom prompt provided, use it
+    if (customPrompt) {
+        logger.debug(`[${functionTag}] Using custom judge prompt`);
+        return customPrompt;
+    }
+    // Build response blocks
+    const responseBlocks = responses
+        .map((r, index) => {
+        const identifier = `response-${index}`;
+        const modelInfo = judge.blindEvaluation
+            ? `Response ${index + 1}`
+            : `${r.provider}/${r.model}`;
+        return `
+<response id="${identifier}">
+<model>${modelInfo}</model>
+<content>
+${r.content}
+</content>
+</response>`;
+    })
+        .join("\n");
+    const criteriaList = judge.criteria
+        .map((c, i) => `${i + 1}. ${c}`)
+        .join("\n");
+    // If synthesis is enabled, judge creates improved response
+    if (judge.synthesizeImprovedResponse) {
+        return `You are an expert AI evaluator and synthesizer. Your task is to:
+1. Evaluate all responses
+2. Synthesize an IMPROVED final response that combines their strengths
+
+USER QUESTION:
+${originalPrompt}
+
+RESPONSES TO EVALUATE:
+${responseBlocks}
+
+EVALUATION CRITERIA:
+${criteriaList}
+
+INSTRUCTIONS:
+1. Score each response on a scale of 0-100 (0 = poor, 100 = excellent)
+2. Consider all evaluation criteria listed above
+3. Provide a ranking of responses from best to worst
+4. Identify the single best response
+5. Provide brief reasoning for your evaluation (max 200 characters)
+6. **SYNTHESIZE an improved response** that:
+   - Combines the best elements from all responses
+   - Addresses any weaknesses identified in the evaluation
+   - Maintains accuracy and technical correctness
+   - Is more complete and higher quality than any single response
+   - Directly answers the user's question (no meta-commentary)
+7. Rate your confidence in this judgment (0.0 to 1.0)
+
+Respond in JSON format:
+{
+  "scores": {
+    "response-0": 85,
+    "response-1": 92
+  },
+  "ranking": ["response-1", "response-0"],
+  "bestResponse": "response-1",
+  "reasoning": "Brief explanation of evaluation",
+  "synthesizedResponse": "Your improved, synthesized response here",
+  "confidenceInJudgment": 0.9
+}`;
+    }
+    // Standard evaluation (no synthesis)
+    return `You are an expert AI evaluator. Evaluate the following responses to the user's question.
+
+USER QUESTION:
+${originalPrompt}
+
+RESPONSES TO EVALUATE:
+${responseBlocks}
+
+EVALUATION CRITERIA:
+${criteriaList}
+
+INSTRUCTIONS:
+1. Score each response on a scale of 0-100 (0 = poor, 100 = excellent)
+2. Consider all evaluation criteria listed above
+3. Provide a ranking of responses from best to worst
+4. Identify the single best response
+5. Provide brief reasoning for your evaluation (max 200 characters)
+6. Rate your confidence in this judgment (0.0 to 1.0)
+
+Respond in JSON format:
+{
+  "scores": {
+    "response-0": 85,
+    "response-1": 92
+  },
+  "ranking": ["response-1", "response-0"],
+  "bestResponse": "response-1",
+  "reasoning": "Brief explanation of evaluation",
+  "confidenceInJudgment": 0.9
+}`;
+}
+/**
+ * Parse judge response to extract scores
+ * @param content - Raw judge response content
+ * @param responses - Original ensemble responses
+ * @param _judge - Judge configuration (unused)
+ * @returns Parsed judge response with scores
+ */
+function parseJudgeResponse(content, responses, _judge) {
+    try {
+        // Try to extract JSON from response
+        const jsonMatch = content.match(/\{[\s\S]*\}/);
+        if (!jsonMatch) {
+            logger.warn(`[${functionTag}] No JSON found in judge response`);
+            return createFallbackScores(responses);
+        }
+        const parsed = JSON.parse(jsonMatch[0]);
+        // Validate and normalize scores to 0-100 range
+        const scores = {};
+        Object.keys(parsed.scores || {}).forEach((key) => {
+            const score = Number(parsed.scores[key]);
+            scores[key] = Math.max(0, Math.min(100, score));
+        });
+        // Ensure all responses have scores
+        responses.forEach((_, index) => {
+            const key = `response-${index}`;
+            if (!(key in scores)) {
+                scores[key] = 50; // Default neutral score
+            }
+        });
+        return {
+            scores,
+            ranking: parsed.ranking || generateRankingFromScores(scores),
+            bestResponse: parsed.bestResponse || findBestResponse(scores),
+            reasoning: truncateReasoning(parsed.reasoning || "No reasoning provided"),
+            synthesizedResponse: parsed.synthesizedResponse, // Extract synthesized response if present
+            confidenceInJudgment: normalizeConfidence(parsed.confidenceInJudgment),
+        };
+    }
+    catch (error) {
+        logger.warn(`[${functionTag}] Failed to parse judge response`, {
+            error: error.message,
+        });
+        return createFallbackScores(responses);
+    }
+}
+/**
+ * Create fallback scores when parsing fails
+ * @param responses - Ensemble responses
+ * @returns Default scores with equal values
+ */
+function createFallbackScores(responses) {
+    const scores = {};
+    const ranking = [];
+    responses.forEach((_, index) => {
+        const key = `response-${index}`;
+        scores[key] = 50; // Neutral score
+        ranking.push(key);
+    });
+    return {
+        scores,
+        ranking,
+        bestResponse: ranking[0],
+        reasoning: "Unable to parse judge evaluation",
+        confidenceInJudgment: 0.5,
+    };
+}
+/**
+ * Generate ranking from scores
+ * @param scores - Score record
+ * @returns Array of response IDs sorted by score descending
+ */
+function generateRankingFromScores(scores) {
+    return Object.keys(scores).sort((a, b) => scores[b] - scores[a]);
+}
+/**
+ * Find best response from scores
+ * @param scores - Score record
+ * @returns Response ID with highest score
+ */
+function findBestResponse(scores) {
+    let bestId = "";
+    let bestScore = -1;
+    Object.keys(scores).forEach((key) => {
+        if (scores[key] > bestScore) {
+            bestScore = scores[key];
+            bestId = key;
+        }
+    });
+    return bestId || Object.keys(scores)[0];
+}
+/**
+ * Truncate reasoning to max 200 characters
+ * @param reasoning - Reasoning text
+ * @returns Truncated reasoning
+ */
+function truncateReasoning(reasoning) {
+    if (reasoning.length <= MAX_REASONING_LENGTH) {
+        return reasoning;
+    }
+    return reasoning.substring(0, MAX_REASONING_LENGTH - 3) + "...";
+}
+/**
+ * Normalize confidence to 0-1 range
+ * @param confidence - Confidence value
+ * @returns Normalized confidence between 0 and 1
+ */
+function normalizeConfidence(confidence) {
+    if (typeof confidence !== "number") {
+        return 0.5;
+    }
+    return Math.max(0, Math.min(1, confidence));
+}
+/**
+ * Aggregate multiple judge scores
+ * @param judgeResults - Array of judge score results
+ * @param _strategy - Aggregation strategy (currently only 'average')
+ * @returns Aggregated scores and ranking
+ */
+function aggregateJudgeScores(judgeResults, _strategy) {
+    // Collect all response IDs
+    const responseIds = new Set();
+    judgeResults.forEach((result) => {
+        Object.keys(result.scores).forEach((id) => responseIds.add(id));
+    });
+    // Calculate average scores
+    const averageScores = {};
+    responseIds.forEach((id) => {
+        const scores = judgeResults
+            .map((result) => result.scores[id])
+            .filter((score) => score !== undefined);
+        if (scores.length > 0) {
+            averageScores[id] =
+                scores.reduce((sum, score) => sum + score, 0) / scores.length;
+        }
+        else {
+            averageScores[id] = 50; // Default
+        }
+    });
+    // Generate ranking from average scores
+    const ranking = generateRankingFromScores(averageScores);
+    const bestResponse = ranking[0];
+    // Calculate aggregate confidence
+    const confidences = judgeResults
+        .map((r) => r.confidenceInJudgment || 0.5)
+        .filter((c) => c > 0);
+    const confidence = confidences.length > 0
+        ? confidences.reduce((sum, c) => sum + c, 0) / confidences.length
+        : 0.5;
+    // Aggregate reasoning
+    const reasoning = `Aggregated from ${judgeResults.length} judges`;
+    return {
+        averageScores,
+        ranking,
+        bestResponse,
+        confidence,
+        reasoning,
+    };
+}
+/**
+ * Calculate consensus level between judges
+ * @param judgeResults - Array of judge score results
+ * @returns Consensus level between 0 and 1
+ */
+function calculateConsensusLevel(judgeResults) {
+    if (judgeResults.length < 2) {
+        return 1.0; // Perfect consensus with single judge
+    }
+    // Calculate agreement on best response
+    const bestResponses = judgeResults.map((r) => r.bestResponse);
+    const modeCounts = new Map();
+    bestResponses.forEach((response) => {
+        if (response) {
+            modeCounts.set(response, (modeCounts.get(response) || 0) + 1);
+        }
+    });
+    const maxCount = Math.max(...Array.from(modeCounts.values()));
+    return maxCount / judgeResults.length;
+}
+/**
+ * Create empty judge scores for error cases
+ * @param judge - Judge configuration
+ * @param responses - Ensemble responses
+ * @returns Empty judge scores
+ */
+function createEmptyJudgeScores(judge, responses) {
+    const scores = {};
+    responses.forEach((_, index) => {
+        scores[`response-${index}`] = 50;
+    });
+    return {
+        judgeProvider: judge.provider,
+        judgeModel: judge.model,
+        scores,
+        criteria: judge.criteria,
+        judgeTime: 0,
+        timestamp: new Date().toISOString(),
+    };
+}
+/**
+ * Create empty scores for error cases
+ * @param judge - Judge configuration
+ * @param responses - Ensemble responses
+ * @returns Empty judge scores
+ */
+function createEmptyScores(judge, responses) {
+    return createEmptyJudgeScores(judge, responses);
+}
+/**
+ * Get best response from judge scores
+ * @param scores - Judge scores or multi-judge scores
+ * @param responses - Original ensemble responses
+ * @returns Best ensemble response
+ */
+export function getBestResponse(scores, responses) {
+    const bestId = scores.bestResponse;
+    if (!bestId) {
+        return undefined;
+    }
+    const index = parseInt(bestId.replace("response-", ""), 10);
+    return responses[index];
+}
+/**
+ * Get ranked responses
+ * @param scores - Judge scores or multi-judge scores
+ * @param responses - Original ensemble responses
+ * @returns Responses sorted by ranking
+ */
+export function getRankedResponses(scores, responses) {
+    if (!scores.ranking || scores.ranking.length === 0) {
+        return responses;
+    }
+    return scores.ranking
+        .map((id) => {
+        const index = parseInt(id.replace("response-", ""), 10);
+        return responses[index];
+    })
+        .filter((r) => r !== undefined);
+}