@juspay/neurolink 9.36.1 → 9.37.0

package/dist/evaluation/scorers/rule/formatScorer.js

@@ -0,0 +1,470 @@
+/**
+ * @file Format Scorer
+ * Evaluates response format compliance (JSON, markdown, code, etc.)
+ */
+import { logger } from "../../../utils/logger.js";
+import { BaseRuleScorer, DEFAULT_RULE_SCORER_CONFIG, } from "./baseRuleScorer.js";
+/**
+ * Scorer metadata for format
+ */
+const FORMAT_METADATA = {
+    id: "format",
+    name: "Format Validator",
+    description: "Evaluates response format compliance (JSON, markdown, code, lists, etc.)",
+    type: "rule",
+    category: "quality",
+    version: "1.0.0",
+    defaultConfig: {
+        ...DEFAULT_RULE_SCORER_CONFIG,
+        threshold: 0.8,
+    },
+    requiredInputs: ["response"],
+    optionalInputs: ["custom"],
+};
+/**
+ * FormatScorer evaluates response format against expected formats
+ */
+export class FormatScorer extends BaseRuleScorer {
+    _formatConfig;
+    constructor(config) {
+        super(FORMAT_METADATA, config);
+        this._formatConfig = {
+            expectedFormat: "plain",
+            strictFormat: false,
+            ...config,
+        };
+    }
+    /**
+     * Get format-specific configuration
+     */
+    get formatConfig() {
+        return this._formatConfig;
+    }
+    /**
+     * Get rules for this scorer
+     */
+    getRules() {
+        const rules = [];
+        const formats = this._formatConfig.allowedFormats ?? [
+            this._formatConfig.expectedFormat ?? "plain",
+        ];
+        // Main format rule
+        rules.push({
+            id: "format-check",
+            description: `Check format is one of: ${formats.join(", ")}`,
+            type: "custom",
+            params: {
+                formats,
+                strict: this._formatConfig.strictFormat ?? false,
+            },
+            weight: 1.0,
+        });
+        // Add specific requirement rules based on format
+        if (formats.includes("markdown") &&
+            this._formatConfig.markdownRequirements) {
+            rules.push({
+                id: "markdown-requirements",
+                description: "Check markdown structure requirements",
+                type: "custom",
+                params: {
+                    requirements: this._formatConfig.markdownRequirements,
+                },
+                weight: 0.5,
+            });
+        }
+        if ((formats.includes("list") ||
+            formats.includes("numbered-list") ||
+            formats.includes("bullet-list")) &&
+            this._formatConfig.listRequirements) {
+            rules.push({
+                id: "list-requirements",
+                description: "Check list structure requirements",
+                type: "custom",
+                params: {
+                    requirements: this._formatConfig.listRequirements,
+                },
+                weight: 0.5,
+            });
+        }
+        if (formats.includes("json") && this._formatConfig.jsonSchema) {
+            rules.push({
+                id: "json-schema",
+                description: "Validate JSON against schema",
+                type: "custom",
+                params: {
+                    schema: this._formatConfig
+                        .jsonSchema,
+                },
+                weight: 0.5,
+            });
+        }
+        return rules;
+    }
+    /**
+     * Evaluate a single format rule
+     */
+    evaluateRule(rule, input) {
+        switch (rule.id) {
+            case "format-check": {
+                const formats = rule.params.formats;
+                const result = this._validateFormat(input.response, formats);
+                return {
+                    passed: result.isValid,
+                    score: result.isValid ? 1.0 : 0.0,
+                };
+            }
+            case "markdown-requirements": {
+                const requirements = rule.params
+                    .requirements;
+                const result = this._checkMarkdownRequirements(input.response, requirements ?? {});
+                return result;
+            }
+            case "list-requirements": {
+                const requirements = rule.params
+                    .requirements;
+                const result = this._checkListRequirements(input.response, requirements ?? {});
+                return result;
+            }
+            case "json-schema": {
+                const schema = rule.params.schema;
+                const result = this._validateJsonSchema(input.response, schema);
+                return result;
+            }
+            default:
+                return { passed: true, score: 1.0 };
+        }
+    }
+    /**
+     * Validate format against allowed formats
+     */
+    _validateFormat(text, allowedFormats) {
+        const issues = [];
+        // Detect format
+        const detectedFormat = this._detectFormat(text);
+        // Check if detected format is allowed
+        const isValid = allowedFormats.includes(detectedFormat) ||
+            (detectedFormat === "plain" && allowedFormats.includes("plain"));
+        if (!isValid) {
+            issues.push(`Expected format(s): ${allowedFormats.join(", ")}, but detected: ${detectedFormat}`);
+        }
+        return { isValid, detectedFormat, issues };
+    }
+    /**
+     * Detect the format of the text
+     */
+    _detectFormat(text) {
+        const trimmed = text.trim();
+        // Check JSON
+        if (this._isValidJson(trimmed)) {
+            return "json";
+        }
+        // Check YAML (basic detection)
+        if (this._isYaml(trimmed)) {
+            return "yaml";
+        }
+        // Check XML/HTML
+        if (this._isXml(trimmed)) {
+            return trimmed.toLowerCase().includes("<!doctype html") ||
+                trimmed.includes("<html")
+                ? "html"
+                : "xml";
+        }
+        // Check code blocks
+        if (this._hasCodeBlocks(trimmed)) {
+            return "code";
+        }
+        // Check markdown elements
+        if (this._hasMarkdownElements(trimmed)) {
+            return "markdown";
+        }
+        // Check lists
+        if (this._isNumberedList(trimmed)) {
+            return "numbered-list";
+        }
+        if (this._isBulletList(trimmed)) {
+            return "bullet-list";
+        }
+        // Check tables (markdown style)
+        if (this._hasTable(trimmed)) {
+            return "table";
+        }
+        return "plain";
+    }
+    /**
+     * Check if text is valid JSON
+     */
+    _isValidJson(text) {
+        try {
+            const parsed = JSON.parse(text);
+            return typeof parsed === "object" && parsed !== null;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Check if text appears to be YAML
+     */
+    _isYaml(text) {
+        // Basic YAML detection: key: value patterns
+        const lines = text.split("\n");
+        let yamlPatternCount = 0;
+        for (const line of lines) {
+            // Skip empty lines and comments
+            if (line.trim() === "" || line.trim().startsWith("#")) {
+                continue;
+            }
+            // Check for key: value pattern (but not URLs)
+            if (/^[\s-]*[\w_]+:\s*.+/.test(line) && !line.includes("://")) {
+                yamlPatternCount++;
+            }
+        }
+        return yamlPatternCount >= 2;
+    }
+    /**
+     * Check if text is XML
+     */
+    _isXml(text) {
+        return ((text.startsWith("<?xml") || text.startsWith("<")) &&
+            text.endsWith(">") &&
+            /<\/?[a-zA-Z][a-zA-Z0-9]*[^>]{0,1000}>/.test(text.slice(0, 10000)));
+    }
+    /**
+     * Check if text has code blocks
+     */
+    _hasCodeBlocks(text) {
+        return /```[\s\S]{0,10000}?```/.test(text) || /`[^`]{1,1000}`/.test(text);
+    }
+    /**
+     * Check if text has markdown elements
+     */
+    _hasMarkdownElements(text) {
+        // Check for headings, bold, italic, links, etc.
+        const markdownPatterns = [
+            /^#{1,6}\s+.+/m, // Headings
+            /\*\*[^*]+\*\*/, // Bold
+            /\*[^*]+\*/, // Italic
+            /__[^_]+__/, // Bold (underscore)
+            /_[^_]+_/, // Italic (underscore)
+            /\[[^\]]{1,500}\]\([^)]{1,2000}\)/, // Links (bounded)
+            /!\[[^\]]{0,500}\]\([^)]{1,2000}\)/, // Images (bounded)
+            /^>\s+.+/m, // Blockquotes
+            /^-{3,}$/m, // Horizontal rule
+            /^\*{3,}$/m, // Horizontal rule
+        ];
+        return markdownPatterns.some((pattern) => pattern.test(text));
+    }
+    /**
+     * Check if text is a numbered list
+     */
+    _isNumberedList(text) {
+        const lines = text
+            .split("\n")
+            .map((l) => l.trim())
+            .filter((l) => l.length > 0);
+        const numberedLines = lines.filter((l) => /^\d+[.)]\s+/.test(l));
+        return (numberedLines.length >= 2 && numberedLines.length / lines.length > 0.5);
+    }
+    /**
+     * Check if text is a bullet list
+     */
+    _isBulletList(text) {
+        const lines = text
+            .split("\n")
+            .map((l) => l.trim())
+            .filter((l) => l.length > 0);
+        const bulletLines = lines.filter((l) => /^[-*+]\s+/.test(l));
+        return bulletLines.length >= 2 && bulletLines.length / lines.length > 0.5;
+    }
+    /**
+     * Check if text has a table
+     */
+    _hasTable(text) {
+        // Markdown table pattern: | header | header |
+        return /\|.+\|/.test(text) && /\|[-:]+\|/.test(text);
+    }
+    /**
+     * Check markdown-specific requirements
+     */
+    _checkMarkdownRequirements(text, requirements) {
+        let totalChecks = 0;
+        let passedChecks = 0;
+        if (requirements.hasHeadings !== undefined) {
+            totalChecks++;
+            if (/^#{1,6}\s+.+/m.test(text) === requirements.hasHeadings) {
+                passedChecks++;
+            }
+        }
+        if (requirements.hasCodeBlocks !== undefined) {
+            totalChecks++;
+            if (this._hasCodeBlocks(text) === requirements.hasCodeBlocks) {
+                passedChecks++;
+            }
+        }
+        if (requirements.hasLinks !== undefined) {
+            totalChecks++;
+            if (/\[[^\]]{1,500}\]\([^)]{1,2000}\)/.test(text) === requirements.hasLinks) {
+                passedChecks++;
+            }
+        }
+        if (requirements.hasLists !== undefined) {
+            totalChecks++;
+            const hasList = this._isNumberedList(text) || this._isBulletList(text);
+            if (hasList === requirements.hasLists) {
+                passedChecks++;
+            }
+        }
+        if (requirements.minHeadingLevel !== undefined ||
+            requirements.maxHeadingLevel !== undefined) {
+            totalChecks++;
+            const headingMatches = text.match(/^(#{1,6})\s+/gm);
+            if (headingMatches) {
+                const levels = headingMatches.map((m) => m.trim().indexOf(" "));
+                const minLevel = Math.min(...levels);
+                const maxLevel = Math.max(...levels);
+                const minOk = requirements.minHeadingLevel === undefined ||
+                    minLevel >= requirements.minHeadingLevel;
+                const maxOk = requirements.maxHeadingLevel === undefined ||
+                    maxLevel <= requirements.maxHeadingLevel;
+                if (minOk && maxOk) {
+                    passedChecks++;
+                }
+            }
+        }
+        if (totalChecks === 0) {
+            return { passed: true, score: 1.0 };
+        }
+        const score = passedChecks / totalChecks;
+        return { passed: score >= 0.8, score };
+    }
+    /**
+     * Check list-specific requirements
+     */
+    _checkListRequirements(text, requirements) {
+        const lines = text
+            .split("\n")
+            .map((l) => l.trim())
+            .filter((l) => l.length > 0);
+        const listItems = lines.filter((l) => /^(\d+[.)]\s+|[-*+]\s+)/.test(l));
+        let totalChecks = 0;
+        let passedChecks = 0;
+        if (requirements.minItems !== undefined) {
+            totalChecks++;
+            if (listItems.length >= requirements.minItems) {
+                passedChecks++;
+            }
+        }
+        if (requirements.maxItems !== undefined) {
+            totalChecks++;
+            if (listItems.length <= requirements.maxItems) {
+                passedChecks++;
+            }
+        }
+        if (requirements.itemPattern !== undefined) {
+            totalChecks++;
+            if (requirements.itemPattern.length > 100) {
+                logger.warn("[FormatScorer] itemPattern too long, using default");
+            }
+            else if (/(\+|\*|\?)\s*\).*?(\+|\*|\?)/.test(requirements.itemPattern) ||
+                /\(\?.*?\)\s*(\+|\*|\{)/.test(requirements.itemPattern)) {
+                // Detect nested quantifiers that can cause catastrophic backtracking
+                logger.warn("[FormatScorer] itemPattern contains potentially unsafe nested quantifiers");
+            }
+            else {
+                try {
+                    const pattern = new RegExp(requirements.itemPattern);
+                    const matchingItems = listItems.filter((item) => pattern.test(item));
+                    if (matchingItems.length === listItems.length) {
+                        passedChecks++;
+                    }
+                }
+                catch {
+                    logger.warn("[FormatScorer] Invalid itemPattern, using default");
+                }
+            }
+        }
+        if (totalChecks === 0) {
+            return { passed: true, score: 1.0 };
+        }
+        const score = passedChecks / totalChecks;
+        return { passed: score >= 0.8, score };
+    }
+    /**
+     * Validate JSON against schema (basic validation)
+     */
+    _validateJsonSchema(text, _schema) {
+        // First check if it's valid JSON
+        try {
+            JSON.parse(text);
+            // TODO: Implement full JSON Schema validation
+            // For now, just check it's valid JSON
+            return { passed: true, score: 1.0 };
+        }
+        catch {
+            return { passed: false, score: 0.0 };
+        }
+    }
+    /**
+     * Override score to add detailed format analysis
+     */
+    async score(input) {
+        const result = await super.score(input);
+        const detectedFormat = this._detectFormat(input.response);
+        return {
+            ...result,
+            metadata: {
+                ...result.metadata,
+                detectedFormat,
+                expectedFormat: this._formatConfig.expectedFormat ?? "plain",
+                allowedFormats: this._formatConfig.allowedFormats ?? [],
+                strictFormat: this._formatConfig.strictFormat ?? false,
+            },
+        };
+    }
+}
+/**
+ * Factory function for creating FormatScorer instances
+ */
+export async function createFormatScorer(config) {
+    return new FormatScorer(config);
+}
+/**
+ * Pre-configured format scorer presets
+ */
+export const FormatScorerPresets = {
+    /** JSON format */
+    json: () => new FormatScorer({
+        expectedFormat: "json",
+        strictFormat: true,
+    }),
+    /** Markdown format */
+    markdown: () => new FormatScorer({
+        expectedFormat: "markdown",
+    }),
+    /** Markdown with headings required */
+    markdownWithHeadings: () => new FormatScorer({
+        expectedFormat: "markdown",
+        markdownRequirements: {
+            hasHeadings: true,
+            minHeadingLevel: 1,
+            maxHeadingLevel: 3,
+        },
+    }),
+    /** Bullet list format */
+    bulletList: () => new FormatScorer({
+        expectedFormat: "bullet-list",
+    }),
+    /** Numbered list format */
+    numberedList: () => new FormatScorer({
+        expectedFormat: "numbered-list",
+    }),
+    /** Code response */
+    code: () => new FormatScorer({
+        expectedFormat: "code",
+    }),
+    /** Plain text only */
+    plainText: () => new FormatScorer({
+        expectedFormat: "plain",
+        strictFormat: true,
+    }),
+};

package/dist/evaluation/scorers/rule/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * @file Rule Scorers Index
+ * Export all rule-based scorers
+ */
+export { BaseRuleScorer, DEFAULT_RULE_SCORER_CONFIG, } from "./baseRuleScorer.js";
+export { type ContentSimilarityConfig, ContentSimilarityScorer, createContentSimilarityScorer, type SimilarityMetric, } from "./contentSimilarityScorer.js";
+export { type CodeLanguage, createFormatScorer, FormatScorer, type FormatScorerConfig, FormatScorerPresets, type FormatType, } from "./formatScorer.js";
+export { createKeywordCoverageScorer, type KeywordCoverageConfig, KeywordCoverageScorer, } from "./keywordCoverageScorer.js";
+export { createLengthScorer, type LengthConstraintType, LengthScorer, type LengthScorerConfig, LengthScorerPresets, type LengthUnit, } from "./lengthScorer.js";

package/dist/evaluation/scorers/rule/index.js

@@ -0,0 +1,10 @@
+/**
+ * @file Rule Scorers Index
+ * Export all rule-based scorers
+ */
+export { BaseRuleScorer, DEFAULT_RULE_SCORER_CONFIG, } from "./baseRuleScorer.js";
+export { ContentSimilarityScorer, createContentSimilarityScorer, } from "./contentSimilarityScorer.js";
+export { createFormatScorer, FormatScorer, FormatScorerPresets, } from "./formatScorer.js";
+// Rule Scorers
+export { createKeywordCoverageScorer, KeywordCoverageScorer, } from "./keywordCoverageScorer.js";
+export { createLengthScorer, LengthScorer, LengthScorerPresets, } from "./lengthScorer.js";

package/dist/evaluation/scorers/rule/keywordCoverageScorer.d.ts

@@ -0,0 +1,83 @@
+/**
+ * @file Keyword Coverage Scorer
+ * Evaluates how well the response covers expected keywords or topics
+ */
+import type { RuleScorerConfig, ScoreResult, ScorerInput, ScorerRule } from "../../../types/scorerTypes.js";
+import { BaseRuleScorer } from "./baseRuleScorer.js";
+/**
+ * Configuration specific to keyword coverage scoring
+ */
+export type KeywordCoverageConfig = RuleScorerConfig & {
+    /** Keywords to check for */
+    keywords?: string[];
+    /** Minimum coverage ratio (0-1) to pass */
+    minCoverage?: number;
+    /** Whether to use case-insensitive matching */
+    caseInsensitive?: boolean;
+    /** Whether to use word boundary matching */
+    wordBoundary?: boolean;
+    /** Synonyms map for flexible matching */
+    synonyms?: Record<string, string[]>;
+    /** Weight different keywords differently */
+    keywordWeights?: Record<string, number>;
+};
+/**
+ * KeywordCoverageScorer evaluates how well a response covers expected keywords
+ */
+export declare class KeywordCoverageScorer extends BaseRuleScorer {
+    private _keywordConfig;
+    private _dynamicRules;
+    constructor(config?: KeywordCoverageConfig);
+    /**
+     * Build scorer rules from keyword configuration
+     */
+    private _buildRulesFromKeywords;
+    /**
+     * Build scorer rules from a keyword list without mutating instance state.
+     * Returns rules directly so callers can use them locally.
+     */
+    private _buildRulesFromKeywordsList;
+    /**
+     * Get keyword-specific configuration
+     */
+    get keywordConfig(): KeywordCoverageConfig;
+    /**
+     * Get rules for this scorer
+     */
+    getRules(): ScorerRule[];
+    /**
+     * Set keywords dynamically
+     */
+    setKeywords(keywords: string[], weights?: Record<string, number>): void;
+    /**
+     * Extract keywords from context or ground truth if not provided
+     */
+    private _extractKeywordsFromInput;
+    /**
+     * Extract important words from text (simple extraction)
+     */
+    private _extractImportantWords;
+    /**
+     * Evaluate a single keyword rule
+     */
+    evaluateRule(rule: ScorerRule, input: ScorerInput): {
+        passed: boolean;
+        score: number;
+    };
+    /**
+     * Check if a keyword is present in text
+     */
+    private _checkKeywordPresence;
+    /**
+     * Override score to handle dynamic keyword extraction
+     */
+    score(input: ScorerInput): Promise<ScoreResult>;
+    /**
+     * Calculate detailed coverage information
+     */
+    private _calculateCoverageDetails;
+}
+/**
+ * Factory function for creating KeywordCoverageScorer instances
+ */
+export declare function createKeywordCoverageScorer(config?: KeywordCoverageConfig): Promise<KeywordCoverageScorer>;