@juspay/neurolink 9.36.1 → 9.37.0

package/dist/evaluation/scorers/rule/keywordCoverageScorer.js

@@ -0,0 +1,347 @@
+/**
+ * @file Keyword Coverage Scorer
+ * Evaluates how well the response covers expected keywords or topics
+ */
+import { BaseRuleScorer } from "./baseRuleScorer.js";
+/**
+ * Scorer metadata for keyword coverage
+ */
+const KEYWORD_COVERAGE_METADATA = {
+    id: "keyword-coverage",
+    name: "Keyword Coverage",
+    description: "Evaluates how well the response covers expected keywords or topics",
+    type: "rule",
+    category: "quality",
+    version: "1.0.0",
+    defaultConfig: {
+        enabled: true,
+        threshold: 0.7,
+        weight: 1.0,
+        timeout: 1000,
+        retries: 0,
+    },
+    requiredInputs: ["response"],
+    optionalInputs: ["query", "context", "groundTruth", "custom"],
+};
+/**
+ * KeywordCoverageScorer evaluates how well a response covers expected keywords
+ */
+export class KeywordCoverageScorer extends BaseRuleScorer {
+    _keywordConfig;
+    _dynamicRules = [];
+    constructor(config) {
+        super(KEYWORD_COVERAGE_METADATA, config);
+        this._keywordConfig = {
+            minCoverage: 0.7,
+            caseInsensitive: true,
+            wordBoundary: true,
+            keywords: [],
+            ...config,
+        };
+        // Build rules from keywords
+        this._buildRulesFromKeywords();
+    }
+    /**
+     * Build scorer rules from keyword configuration
+     */
+    _buildRulesFromKeywords() {
+        this._dynamicRules = this._buildRulesFromKeywordsList(this._keywordConfig.keywords ?? []);
+    }
+    /**
+     * Build scorer rules from a keyword list without mutating instance state.
+     * Returns rules directly so callers can use them locally.
+     */
+    _buildRulesFromKeywordsList(keywords) {
+        const weights = this._keywordConfig.keywordWeights ?? {};
+        return keywords.map((keyword, index) => ({
+            id: `keyword-${index}-${keyword.toLowerCase().replace(/\s+/g, "-")}`,
+            description: `Check for keyword: ${keyword}`,
+            type: "keyword",
+            params: {
+                keyword,
+                caseInsensitive: this._keywordConfig.caseInsensitive ?? true,
+                wordBoundary: this._keywordConfig.wordBoundary ?? true,
+                synonyms: this._keywordConfig.synonyms?.[keyword] ?? [],
+            },
+            weight: weights[keyword] ?? 1.0,
+        }));
+    }
+    /**
+     * Get keyword-specific configuration
+     */
+    get keywordConfig() {
+        return this._keywordConfig;
+    }
+    /**
+     * Get rules for this scorer
+     */
+    getRules() {
+        return this._dynamicRules;
+    }
+    /**
+     * Set keywords dynamically
+     */
+    setKeywords(keywords, weights) {
+        this._keywordConfig.keywords = keywords;
+        if (weights) {
+            this._keywordConfig.keywordWeights = weights;
+        }
+        this._buildRulesFromKeywords();
+    }
+    /**
+     * Extract keywords from context or ground truth if not provided
+     */
+    _extractKeywordsFromInput(input) {
+        // If keywords are configured, use those
+        if (this._keywordConfig.keywords &&
+            this._keywordConfig.keywords.length > 0) {
+            return this._keywordConfig.keywords;
+        }
+        // Try to extract from custom input
+        if (input.custom?.keywords && Array.isArray(input.custom.keywords)) {
+            return input.custom.keywords;
+        }
+        // Extract important words from ground truth if available
+        if (input.groundTruth) {
+            return this._extractImportantWords(input.groundTruth);
+        }
+        // Extract from context
+        if (input.context && input.context.length > 0) {
+            const contextText = input.context.join(" ");
+            return this._extractImportantWords(contextText);
+        }
+        return [];
+    }
+    /**
+     * Extract important words from text (simple extraction)
+     */
+    _extractImportantWords(text) {
+        // Remove common stop words and extract longer words
+        const stopWords = new Set([
+            "the",
+            "a",
+            "an",
+            "is",
+            "are",
+            "was",
+            "were",
+            "be",
+            "been",
+            "being",
+            "have",
+            "has",
+            "had",
+            "do",
+            "does",
+            "did",
+            "will",
+            "would",
+            "could",
+            "should",
+            "may",
+            "might",
+            "must",
+            "shall",
+            "can",
+            "need",
+            "dare",
+            "ought",
+            "used",
+            "to",
+            "of",
+            "in",
+            "for",
+            "on",
+            "with",
+            "at",
+            "by",
+            "from",
+            "as",
+            "into",
+            "through",
+            "during",
+            "before",
+            "after",
+            "above",
+            "below",
+            "between",
+            "under",
+            "again",
+            "further",
+            "then",
+            "once",
+            "and",
+            "but",
+            "or",
+            "nor",
+            "so",
+            "yet",
+            "both",
+            "either",
+            "neither",
+            "not",
+            "only",
+            "own",
+            "same",
+            "than",
+            "too",
+            "very",
+            "just",
+            "also",
+            "now",
+            "here",
+            "there",
+            "when",
+            "where",
+            "why",
+            "how",
+            "all",
+            "each",
+            "every",
+            "any",
+            "some",
+            "no",
+            "other",
+            "such",
+            "this",
+            "that",
+            "these",
+            "those",
+            "i",
+            "you",
+            "he",
+            "she",
+            "it",
+            "we",
+            "they",
+            "what",
+            "which",
+            "who",
+            "whom",
+        ]);
+        const words = text
+            .toLowerCase()
+            .replace(/[^\w\s]/g, " ")
+            .split(/\s+/)
+            .filter((word) => word.length > 3 && !stopWords.has(word));
+        // Get unique words, sorted by frequency
+        const wordCounts = new Map();
+        for (const word of words) {
+            wordCounts.set(word, (wordCounts.get(word) ?? 0) + 1);
+        }
+        return Array.from(wordCounts.entries())
+            .sort((a, b) => b[1] - a[1])
+            .slice(0, 10) // Top 10 keywords
+            .map(([word]) => word);
+    }
+    /**
+     * Evaluate a single keyword rule
+     */
+    evaluateRule(rule, input) {
+        const keyword = rule.params.keyword;
+        const synonyms = rule.params.synonyms ?? [];
+        const caseInsensitive = rule.params.caseInsensitive;
+        const wordBoundary = rule.params.wordBoundary;
+        const response = input.response;
+        // Check main keyword
+        const found = this._checkKeywordPresence(response, keyword, caseInsensitive, wordBoundary);
+        if (found) {
+            return { passed: true, score: 1.0 };
+        }
+        // Check synonyms
+        for (const synonym of synonyms) {
+            if (this._checkKeywordPresence(response, synonym, caseInsensitive, wordBoundary)) {
+                return { passed: true, score: 0.9 }; // Slightly lower for synonym match
+            }
+        }
+        return { passed: false, score: 0.0 };
+    }
+    /**
+     * Check if a keyword is present in text
+     */
+    _checkKeywordPresence(text, keyword, caseInsensitive, wordBoundary) {
+        if (wordBoundary) {
+            return this.containsKeyword(text, keyword, caseInsensitive);
+        }
+        if (caseInsensitive) {
+            return text.toLowerCase().includes(keyword.toLowerCase());
+        }
+        return text.includes(keyword);
+    }
+    /**
+     * Override score to handle dynamic keyword extraction
+     */
+    async score(input) {
+        // Extract keywords if not configured
+        const keywords = this._extractKeywordsFromInput(input);
+        // Build rules locally without mutating instance state
+        const effectiveRules = this._dynamicRules.length > 0
+            ? this._dynamicRules
+            : this._buildRulesFromKeywordsList(keywords);
+        // If still no keywords, return a passing score with note
+        if (effectiveRules.length === 0) {
+            return this.createScoreResult(10, "No keywords configured or extractable - passing by default", {
+                metadata: {
+                    noKeywordsConfigured: true,
+                },
+            });
+        }
+        // Call parent score method
+        const result = await super.score(input);
+        // Add coverage details to metadata
+        const details = this._calculateCoverageDetails(input);
+        return {
+            ...result,
+            metadata: {
+                ...result.metadata,
+                coverageDetails: details,
+            },
+        };
+    }
+    /**
+     * Calculate detailed coverage information
+     */
+    _calculateCoverageDetails(input) {
+        const keywords = this._keywordConfig.keywords ?? [];
+        const weights = this._keywordConfig.keywordWeights ?? {};
+        const response = input.response;
+        const foundKeywords = [];
+        const missingKeywords = [];
+        let totalWeight = 0;
+        let foundWeight = 0;
+        for (const keyword of keywords) {
+            const weight = weights[keyword] ?? 1.0;
+            totalWeight += weight;
+            const found = this._checkKeywordPresence(response, keyword, this._keywordConfig.caseInsensitive ?? true, this._keywordConfig.wordBoundary ?? true);
+            if (found) {
+                foundKeywords.push(keyword);
+                foundWeight += weight;
+            }
+            else {
+                // Check synonyms
+                const synonyms = this._keywordConfig.synonyms?.[keyword] ?? [];
+                const synonymFound = synonyms.some((syn) => this._checkKeywordPresence(response, syn, this._keywordConfig.caseInsensitive ?? true, this._keywordConfig.wordBoundary ?? true));
+                if (synonymFound) {
+                    foundKeywords.push(keyword);
+                    foundWeight += weight * 0.9; // Slightly less for synonym
+                }
+                else {
+                    missingKeywords.push(keyword);
+                }
+            }
+        }
+        return {
+            totalKeywords: keywords.length,
+            foundKeywords,
+            missingKeywords,
+            coverageRatio: keywords.length > 0 ? foundKeywords.length / keywords.length : 1,
+            weightedCoverage: totalWeight > 0 ? foundWeight / totalWeight : 1,
+        };
+    }
+}
+/**
+ * Factory function for creating KeywordCoverageScorer instances
+ */
+export async function createKeywordCoverageScorer(config) {
+    return new KeywordCoverageScorer(config);
+}

package/dist/evaluation/scorers/rule/lengthScorer.d.ts

@@ -0,0 +1,105 @@
+/**
+ * @file Length Scorer
+ * Evaluates response length against configured constraints
+ */
+import type { RuleScorerConfig, ScoreResult, ScorerInput, ScorerRule } from "../../../types/scorerTypes.js";
+import { BaseRuleScorer } from "./baseRuleScorer.js";
+/**
+ * Length measurement unit
+ */
+export type LengthUnit = "words" | "characters" | "sentences" | "paragraphs" | "tokens";
+/**
+ * Length constraint type
+ */
+export type LengthConstraintType = "exact" | "range" | "minimum" | "maximum" | "ratio";
+/**
+ * Configuration specific to length scoring
+ */
+export type LengthScorerConfig = RuleScorerConfig & {
+    /** Unit of measurement */
+    unit?: LengthUnit;
+    /** Constraint type */
+    constraintType?: LengthConstraintType;
+    /** Minimum length (for range/minimum constraints) */
+    minLength?: number;
+    /** Maximum length (for range/maximum constraints) */
+    maxLength?: number;
+    /** Exact length (for exact constraint) */
+    exactLength?: number;
+    /** Tolerance for exact length (as percentage) */
+    tolerance?: number;
+    /** Compare ratio with query/context length */
+    ratioTarget?: number;
+    /** Ratio reference: compare against query or context */
+    ratioReference?: "query" | "context";
+    /** Scoring mode: binary (pass/fail) or proportional */
+    scoringMode?: "binary" | "proportional";
+};
+/**
+ * LengthScorer evaluates response length against configurable constraints
+ */
+export declare class LengthScorer extends BaseRuleScorer {
+    private _lengthConfig;
+    constructor(config?: LengthScorerConfig);
+    /**
+     * Get length-specific configuration
+     */
+    get lengthConfig(): LengthScorerConfig;
+    /**
+     * Get rules for this scorer
+     */
+    getRules(): ScorerRule[];
+    /**
+     * Measure text length in various units
+     */
+    private _measureLength;
+    /**
+     * Get length in the configured unit
+     */
+    private _getLengthInUnit;
+    /**
+     * Count sentences in text
+     */
+    private _countSentences;
+    /**
+     * Count paragraphs in text
+     */
+    private _countParagraphs;
+    /**
+     * Estimate token count (rough approximation)
+     * GPT-style: ~4 characters per token on average
+     */
+    private _estimateTokens;
+    /**
+     * Evaluate a single length rule
+     */
+    evaluateRule(rule: ScorerRule, input: ScorerInput): {
+        passed: boolean;
+        score: number;
+    };
+    /**
+     * Override score to add detailed length metrics
+     */
+    score(input: ScorerInput): Promise<ScoreResult>;
+}
+/**
+ * Factory function for creating LengthScorer instances
+ */
+export declare function createLengthScorer(config?: LengthScorerConfig): Promise<LengthScorer>;
+/**
+ * Pre-configured length scorer presets
+ */
+export declare const LengthScorerPresets: {
+    /** Short response (50-150 words) */
+    readonly short: () => LengthScorer;
+    /** Medium response (100-300 words) */
+    readonly medium: () => LengthScorer;
+    /** Long response (200-500 words) */
+    readonly long: () => LengthScorer;
+    /** Concise response (max 100 words) */
+    readonly concise: () => LengthScorer;
+    /** Detailed response (min 300 words) */
+    readonly detailed: () => LengthScorer;
+    /** Tweet-length (max 280 characters) */
+    readonly tweet: () => LengthScorer;
+};