@juspay/neurolink 9.36.1 → 9.38.0

package/dist/evaluation/scorers/scorerBuilder.js

@@ -0,0 +1,420 @@
+/**
+ * @file Scorer Builder
+ * Fluent builder API for creating custom scorers
+ */
+import { composeScorers, createFunctionScorer, } from "./customScorerUtils.js";
+/**
+ * Fluent builder for creating custom scorers
+ */
+export class ScorerBuilder {
+    _id;
+    _name;
+    _description;
+    _type = "rule";
+    _category = "custom";
+    _version = "1.0.0";
+    _requiredInputs = ["response"];
+    _optionalInputs = [
+        "query",
+        "context",
+        "groundTruth",
+    ];
+    _threshold = 0.7;
+    _weight = 1.0;
+    _timeout = 5000;
+    _retries = 0;
+    _scorerFn;
+    _rules = [];
+    _subScorers = [];
+    _aggregation = "average";
+    _subScorerWeights = [];
+    constructor(id, name) {
+        this._id = id;
+        this._name = name;
+    }
+    /**
+     * Create a new scorer builder
+     */
+    static create(id, name) {
+        return new ScorerBuilder(id, name);
+    }
+    /**
+     * Set scorer description
+     */
+    description(desc) {
+        this._description = desc;
+        return this;
+    }
+    /**
+     * Set scorer type
+     */
+    type(type) {
+        this._type = type;
+        return this;
+    }
+    /**
+     * Set scorer category
+     */
+    category(category) {
+        this._category = category;
+        return this;
+    }
+    /**
+     * Set scorer version
+     */
+    version(version) {
+        this._version = version;
+        return this;
+    }
+    /**
+     * Set required inputs
+     */
+    requireInputs(...inputs) {
+        this._requiredInputs = inputs;
+        return this;
+    }
+    /**
+     * Set optional inputs
+     */
+    optionalInputs(...inputs) {
+        this._optionalInputs = inputs;
+        return this;
+    }
+    /**
+     * Set pass/fail threshold
+     */
+    threshold(threshold) {
+        this._threshold = threshold;
+        return this;
+    }
+    /**
+     * Set weight for aggregation
+     */
+    weight(weight) {
+        this._weight = weight;
+        return this;
+    }
+    /**
+     * Set execution timeout
+     */
+    timeout(ms) {
+        this._timeout = ms;
+        return this;
+    }
+    /**
+     * Set retry count
+     */
+    retries(count) {
+        this._retries = count;
+        return this;
+    }
+    /**
+     * Set the scoring function
+     */
+    scoringFunction(fn) {
+        this._scorerFn = fn;
+        return this;
+    }
+    /**
+     * Add a sub-scorer for composition
+     */
+    addScorer(scorer, weight) {
+        this._subScorers.push(scorer);
+        this._subScorerWeights.push(weight ?? 1.0);
+        return this;
+    }
+    /**
+     * Set aggregation method for composed scorers
+     */
+    aggregateWith(method) {
+        this._aggregation = method;
+        return this;
+    }
+    /**
+     * Add a regex check rule
+     */
+    matchesPattern(pattern, options) {
+        const patternStr = typeof pattern === "string" ? pattern : pattern.source;
+        const flags = typeof pattern === "string" ? "gi" : pattern.flags;
+        // Validate pattern safety (same guards as regex scorer utilities)
+        if (patternStr.length > 200) {
+            throw new Error("Regex pattern exceeds maximum length of 200 characters");
+        }
+        if (/(\+|\*|\{)\S*(\+|\*|\{)/.test(patternStr)) {
+            throw new Error("Regex pattern contains nested quantifiers which may cause catastrophic backtracking");
+        }
+        // Pre-compile to validate the pattern at build time
+        try {
+            new RegExp(patternStr, flags);
+        }
+        catch (e) {
+            throw new Error(`Invalid regex pattern: ${e instanceof Error ? e.message : String(e)}`, { cause: e });
+        }
+        this._rules.push({
+            id: options?.id ?? `regex-${this._rules.length}`,
+            description: `Match pattern: ${patternStr}`,
+            type: "regex",
+            params: {
+                pattern: patternStr,
+                flags,
+            },
+            weight: options?.weight ?? 1.0,
+        });
+        return this;
+    }
+    /**
+     * Add a keyword check rule
+     */
+    containsKeyword(keyword, options) {
+        this._rules.push({
+            id: options?.id ?? `keyword-${this._rules.length}`,
+            description: `Contains keyword: ${keyword}`,
+            type: "keyword",
+            params: {
+                keyword,
+                caseInsensitive: true,
+            },
+            weight: options?.weight ?? 1.0,
+        });
+        return this;
+    }
+    /**
+     * Add a length check rule
+     */
+    hasLength(options) {
+        this._rules.push({
+            id: options.id ?? `length-${this._rules.length}`,
+            description: "Length check",
+            type: "length",
+            params: {
+                minWords: options.minWords ?? null,
+                maxWords: options.maxWords ?? null,
+                minChars: options.minChars ?? null,
+                maxChars: options.maxChars ?? null,
+            },
+            weight: options.weight ?? 1.0,
+        });
+        return this;
+    }
+    /**
+     * Add a custom rule
+     */
+    customRule(rule) {
+        this._rules.push(rule);
+        return this;
+    }
+    /**
+     * Build the scorer
+     */
+    build() {
+        // If we have sub-scorers, create a composed scorer
+        if (this._subScorers.length > 0) {
+            return composeScorers(this._id, this._name, this._subScorers, {
+                aggregation: this._aggregation,
+                weights: this._subScorerWeights,
+                description: this._description,
+                config: this._buildConfig(),
+            });
+        }
+        // If we have a scoring function, use it
+        if (this._scorerFn) {
+            return createFunctionScorer(this._id, this._name, this._scorerFn, {
+                type: this._type,
+                version: this._version,
+                requiredInputs: this._requiredInputs,
+                optionalInputs: this._optionalInputs,
+                description: this._description,
+                category: this._category,
+                config: this._buildConfig(),
+            });
+        }
+        // If we have rules, create a rule-based scorer
+        if (this._rules.length > 0) {
+            return this._buildRuleScorer();
+        }
+        // Default: return a pass-through scorer
+        return createFunctionScorer(this._id, this._name, async () => ({
+            score: 10,
+            reasoning: "No scoring logic defined - passing by default",
+        }), {
+            type: this._type,
+            version: this._version,
+            requiredInputs: this._requiredInputs,
+            optionalInputs: this._optionalInputs,
+            description: this._description,
+            category: this._category,
+            config: this._buildConfig(),
+        });
+    }
+    /**
+     * Build configuration object
+     */
+    _buildConfig() {
+        return {
+            enabled: true,
+            threshold: this._threshold,
+            weight: this._weight,
+            timeout: this._timeout,
+            retries: this._retries,
+        };
+    }
+    /**
+     * Build a rule-based scorer from accumulated rules
+     */
+    _buildRuleScorer() {
+        const rules = this._rules;
+        const config = this._buildConfig();
+        return createFunctionScorer(this._id, this._name, async (input) => {
+            const results = [];
+            for (const rule of rules) {
+                const result = this._evaluateRule(rule, input);
+                results.push({ rule, ...result });
+            }
+            // Calculate weighted score
+            let totalWeight = 0;
+            let weightedScore = 0;
+            for (const result of results) {
+                const weight = result.rule.weight ?? 1.0;
+                totalWeight += weight;
+                weightedScore += result.score * weight;
+            }
+            const finalScore = totalWeight > 0 ? (weightedScore / totalWeight) * 10 : 10;
+            const passedCount = results.filter((r) => r.passed).length;
+            return {
+                score: finalScore,
+                reasoning: `${passedCount}/${rules.length} rules passed`,
+                metadata: {
+                    ruleResults: results.map((r) => ({
+                        ruleId: r.rule.id,
+                        passed: r.passed,
+                        score: r.score,
+                    })),
+                },
+            };
+        }, {
+            type: this._type,
+            version: this._version,
+            requiredInputs: this._requiredInputs,
+            optionalInputs: this._optionalInputs,
+            description: this._description,
+            category: this._category,
+            config,
+        });
+    }
+    /**
+     * Evaluate a single rule
+     */
+    _evaluateRule(rule, input) {
+        switch (rule.type) {
+            case "regex": {
+                let regex;
+                try {
+                    regex = new RegExp(rule.params.pattern, rule.params.flags ?? "i");
+                }
+                catch {
+                    return { passed: false, score: 0.0 };
+                }
+                if (regex.global) {
+                    regex.lastIndex = 0;
+                }
+                const matches = regex.test(input.response);
+                return { passed: matches, score: matches ? 1.0 : 0.0 };
+            }
+            case "keyword": {
+                const keyword = rule.params.keyword;
+                const caseInsensitive = rule.params.caseInsensitive;
+                const text = caseInsensitive
+                    ? input.response.toLowerCase()
+                    : input.response;
+                const search = caseInsensitive ? keyword.toLowerCase() : keyword;
+                const found = text.includes(search);
+                return { passed: found, score: found ? 1.0 : 0.0 };
+            }
+            case "length": {
+                const wordCount = input.response
+                    .trim()
+                    .split(/\s+/)
+                    .filter((w) => w.length > 0).length;
+                const charCount = input.response.length;
+                const minWords = rule.params.minWords;
+                const maxWords = rule.params.maxWords;
+                const minChars = rule.params.minChars;
+                const maxChars = rule.params.maxChars;
+                let passed = true;
+                if (minWords !== null && wordCount < minWords) {
+                    passed = false;
+                }
+                if (maxWords !== null && wordCount > maxWords) {
+                    passed = false;
+                }
+                if (minChars !== null && charCount < minChars) {
+                    passed = false;
+                }
+                if (maxChars !== null && charCount > maxChars) {
+                    passed = false;
+                }
+                return { passed, score: passed ? 1.0 : 0.0 };
+            }
+            case "custom": {
+                // Execute custom evaluator if provided
+                if (rule.params?.evaluate &&
+                    typeof rule.params.evaluate === "function") {
+                    try {
+                        const customResult = rule.params.evaluate(input);
+                        if (typeof customResult?.passed === "boolean" &&
+                            typeof customResult?.score === "number") {
+                            return customResult;
+                        }
+                    }
+                    catch {
+                        return { passed: false, score: 0.0 };
+                    }
+                }
+                // No evaluator provided - pass by default with warning
+                return { passed: true, score: 1.0 };
+            }
+            default:
+                return { passed: true, score: 1.0 };
+        }
+    }
+}
+/**
+ * Quick builder factory functions
+ */
+export const Scorers = {
+    /**
+     * Create a new scorer builder
+     */
+    create: (id, name) => ScorerBuilder.create(id, name),
+    /**
+     * Create a simple pass/fail scorer based on a condition
+     */
+    passIf: (id, name, condition) => ScorerBuilder.create(id, name).scoringFunction(async (input) => ({
+        score: condition(input) ? 10 : 0,
+        reasoning: condition(input) ? "Condition passed" : "Condition failed",
+    })),
+    /**
+     * Create a scorer that checks for required content
+     */
+    requiresContent: (id, name, keywords) => {
+        const builder = ScorerBuilder.create(id, name).category("quality");
+        for (const keyword of keywords) {
+            builder.containsKeyword(keyword);
+        }
+        return builder;
+    },
+    /**
+     * Create a scorer with length constraints
+     */
+    withLength: (id, name, options) => ScorerBuilder.create(id, name).category("quality").hasLength(options),
+    /**
+     * Create a scorer that combines multiple scorers
+     */
+    combine: (id, name, scorers) => {
+        const builder = ScorerBuilder.create(id, name).category("custom");
+        for (const scorer of scorers) {
+            builder.addScorer(scorer);
+        }
+        return builder;
+    },
+};

package/dist/evaluation/scorers/scorerRegistry.d.ts

@@ -0,0 +1,62 @@
+/**
+ * @file Scorer registry for managing scorer registration and discovery
+ * Follows NeuroLink's factory + registry pattern with dynamic imports
+ */
+import type { Scorer, ScorerCategory, ScorerConfig, ScorerFactory, ScorerMetadata, ScorerRegistryEntry, ScorerType } from "../../types/scorerTypes.js";
+/**
+ * Central registry for all scorers
+ * Manages registration, discovery, and instantiation
+ */
+export declare class ScorerRegistry {
+    private static scorers;
+    private static initialized;
+    private static initPromise;
+    /**
+     * Register a scorer with the registry
+     */
+    static register(entry: ScorerRegistryEntry): void;
+    /**
+     * Register a scorer using a simple configuration
+     */
+    static registerScorer(metadata: ScorerMetadata, factory: ScorerFactory, aliases?: string[]): void;
+    /**
+     * Register built-in scorers using dynamic imports
+     */
+    static registerBuiltInScorers(): Promise<void>;
+    /**
+     * Get a scorer instance by ID
+     */
+    static getScorer(scorerId: string, config?: ScorerConfig): Promise<Scorer | undefined>;
+    /**
+     * Get scorers by category
+     */
+    static getScorersByCategory(category: ScorerCategory): ScorerRegistryEntry[];
+    /**
+     * Get scorers by type
+     */
+    static getScorersByType(type: ScorerType): ScorerRegistryEntry[];
+    /**
+     * List all registered scorer metadata
+     */
+    static list(): ScorerMetadata[];
+    /**
+     * Check if a scorer is registered
+     */
+    static has(scorerId: string): boolean;
+    /**
+     * Unregister a scorer
+     */
+    static unregister(scorerId: string): boolean;
+    /**
+     * Clear all registered scorers
+     */
+    static clear(): void;
+    /**
+     * Ensure built-in scorers are initialized
+     */
+    private static ensureInitialized;
+    /**
+     * Get the number of registered scorers (excluding aliases)
+     */
+    static get size(): number;
+}