@memberjunction/testing-engine 0.0.1 → 2.118.0

package/dist/oracles/ExactMatchOracle.d.ts

@@ -0,0 +1,98 @@
+/**
+ * @fileoverview Exact match oracle implementation
+ * @module @memberjunction/testing-engine
+ */
+import { IOracle } from './IOracle';
+import { OracleInput, OracleConfig, OracleResult } from '../types';
+/**
+ * Exact Match Oracle.
+ *
+ * Performs deterministic comparison between expected and actual output.
+ * Supports various comparison modes (exact, contains, regex, deep equality).
+ *
+ * Configuration:
+ * - mode: Comparison mode ('exact' | 'contains' | 'regex' | 'deep' | 'partial')
+ * - caseSensitive: Whether string comparisons are case-sensitive (default: true)
+ * - ignoreWhitespace: Whether to normalize whitespace before comparison (default: false)
+ * - fields: For 'partial' mode, which fields to compare (array of paths)
+ *
+ * @example
+ * ```typescript
+ * const oracle = new ExactMatchOracle();
+ *
+ * // Exact match
+ * const result1 = await oracle.evaluate({
+ *     expectedOutput: { status: 'success', count: 5 },
+ *     actualOutput: { status: 'success', count: 5 }
+ * }, { mode: 'exact' });
+ *
+ * // Contains mode (actual must contain all expected fields)
+ * const result2 = await oracle.evaluate({
+ *     expectedOutput: { status: 'success' },
+ *     actualOutput: { status: 'success', count: 5, extra: 'data' }
+ * }, { mode: 'contains' });
+ *
+ * // Regex match
+ * const result3 = await oracle.evaluate({
+ *     expectedOutput: { pattern: 'sales.*region' },
+ *     actualOutput: { response: 'Sales by region report' }
+ * }, { mode: 'regex' });
+ * ```
+ */
+export declare class ExactMatchOracle implements IOracle {
+    readonly type = "exact-match";
+    /**
+     * Evaluate exact match between expected and actual output.
+     *
+     * @param input - Oracle input with expected and actual output
+     * @param config - Oracle configuration
+     * @returns Oracle result with match details
+     */
+    evaluate(input: OracleInput, config: OracleConfig): Promise<OracleResult>;
+    /**
+     * Exact JSON string match.
+     * @private
+     */
+    private exactMatch;
+    /**
+     * Check if actual contains all expected fields.
+     * @private
+     */
+    private containsMatch;
+    /**
+     * Find missing fields in actual compared to expected.
+     * @private
+     */
+    private findMissingFields;
+    /**
+     * Regex pattern matching.
+     * @private
+     */
+    private regexMatch;
+    /**
+     * Extract regex patterns from expected output.
+     * @private
+     */
+    private extractPatterns;
+    /**
+     * Deep equality check.
+     * @private
+     */
+    private deepEqual;
+    /**
+     * Recursive deep equality check.
+     * @private
+     */
+    private deepEquality;
+    /**
+     * Partial match on specific fields.
+     * @private
+     */
+    private partialMatch;
+    /**
+     * Get field value by path (e.g., 'user.name' or 'items[0].id').
+     * @private
+     */
+    private getFieldValue;
+}
+//# sourceMappingURL=ExactMatchOracle.d.ts.map

package/dist/oracles/ExactMatchOracle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ExactMatchOracle.d.ts","sourceRoot":"","sources":["../../src/oracles/ExactMatchOracle.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAEnE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,qBAAa,gBAAiB,YAAW,OAAO;IAC5C,QAAQ,CAAC,IAAI,iBAAiB;IAE9B;;;;;;OAMG;IACG,QAAQ,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IA+E/E;;;OAGG;IACH,OAAO,CAAC,UAAU;IA0BlB;;;OAGG;IACH,OAAO,CAAC,aAAa;IAsBrB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAkCzB;;;OAGG;IACH,OAAO,CAAC,UAAU;IAoClB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAiBvB;;;OAGG;IACH,OAAO,CAAC,SAAS;IAajB;;;OAGG;IACH,OAAO,CAAC,YAAY;IAwCpB;;;OAGG;IACH,OAAO,CAAC,YAAY;IAwCpB;;;OAGG;IACH,OAAO,CAAC,aAAa;CAgCxB"}

package/dist/oracles/ExactMatchOracle.js

@@ -0,0 +1,355 @@
+"use strict";
+/**
+ * @fileoverview Exact match oracle implementation
+ * @module @memberjunction/testing-engine
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ExactMatchOracle = void 0;
+/**
+ * Exact Match Oracle.
+ *
+ * Performs deterministic comparison between expected and actual output.
+ * Supports various comparison modes (exact, contains, regex, deep equality).
+ *
+ * Configuration:
+ * - mode: Comparison mode ('exact' | 'contains' | 'regex' | 'deep' | 'partial')
+ * - caseSensitive: Whether string comparisons are case-sensitive (default: true)
+ * - ignoreWhitespace: Whether to normalize whitespace before comparison (default: false)
+ * - fields: For 'partial' mode, which fields to compare (array of paths)
+ *
+ * @example
+ * ```typescript
+ * const oracle = new ExactMatchOracle();
+ *
+ * // Exact match
+ * const result1 = await oracle.evaluate({
+ *     expectedOutput: { status: 'success', count: 5 },
+ *     actualOutput: { status: 'success', count: 5 }
+ * }, { mode: 'exact' });
+ *
+ * // Contains mode (actual must contain all expected fields)
+ * const result2 = await oracle.evaluate({
+ *     expectedOutput: { status: 'success' },
+ *     actualOutput: { status: 'success', count: 5, extra: 'data' }
+ * }, { mode: 'contains' });
+ *
+ * // Regex match
+ * const result3 = await oracle.evaluate({
+ *     expectedOutput: { pattern: 'sales.*region' },
+ *     actualOutput: { response: 'Sales by region report' }
+ * }, { mode: 'regex' });
+ * ```
+ */
+class ExactMatchOracle {
+    constructor() {
+        this.type = 'exact-match';
+    }
+    /**
+     * Evaluate exact match between expected and actual output.
+     *
+     * @param input - Oracle input with expected and actual output
+     * @param config - Oracle configuration
+     * @returns Oracle result with match details
+     */
+    async evaluate(input, config) {
+        try {
+            const mode = config.mode || 'exact';
+            const caseSensitive = config.caseSensitive !== false; // Default true
+            const ignoreWhitespace = config.ignoreWhitespace === true; // Default false
+            const expected = input.expectedOutput;
+            const actual = input.actualOutput;
+            if (!expected) {
+                return {
+                    oracleType: this.type,
+                    passed: false,
+                    score: 0,
+                    message: 'No expected output provided'
+                };
+            }
+            if (!actual) {
+                return {
+                    oracleType: this.type,
+                    passed: false,
+                    score: 0,
+                    message: 'No actual output provided'
+                };
+            }
+            // Perform comparison based on mode
+            let result;
+            switch (mode) {
+                case 'exact':
+                    result = this.exactMatch(expected, actual);
+                    break;
+                case 'contains':
+                    result = this.containsMatch(expected, actual);
+                    break;
+                case 'regex':
+                    result = this.regexMatch(expected, actual, caseSensitive);
+                    break;
+                case 'deep':
+                    result = this.deepEqual(expected, actual);
+                    break;
+                case 'partial':
+                    result = this.partialMatch(expected, actual, config.fields);
+                    break;
+                default:
+                    return {
+                        oracleType: this.type,
+                        passed: false,
+                        score: 0,
+                        message: `Unknown comparison mode: ${mode}`
+                    };
+            }
+            return {
+                oracleType: this.type,
+                ...result
+            };
+        }
+        catch (error) {
+            return {
+                oracleType: this.type,
+                passed: false,
+                score: 0,
+                message: `Exact match error: ${error.message}`
+            };
+        }
+    }
+    /**
+     * Exact JSON string match.
+     * @private
+     */
+    exactMatch(expected, actual) {
+        const expectedStr = JSON.stringify(expected);
+        const actualStr = JSON.stringify(actual);
+        if (expectedStr === actualStr) {
+            return {
+                passed: true,
+                score: 1.0,
+                message: 'Output exactly matches expected'
+            };
+        }
+        else {
+            return {
+                passed: false,
+                score: 0,
+                message: 'Output does not match expected',
+                details: {
+                    expected: expectedStr.substring(0, 200),
+                    actual: actualStr.substring(0, 200)
+                }
+            };
+        }
+    }
+    /**
+     * Check if actual contains all expected fields.
+     * @private
+     */
+    containsMatch(expected, actual) {
+        const missingFields = this.findMissingFields(expected, actual, '');
+        if (missingFields.length === 0) {
+            return {
+                passed: true,
+                score: 1.0,
+                message: 'Output contains all expected fields'
+            };
+        }
+        else {
+            return {
+                passed: false,
+                score: 0,
+                message: `Missing ${missingFields.length} expected field(s)`,
+                details: { missingFields }
+            };
+        }
+    }
+    /**
+     * Find missing fields in actual compared to expected.
+     * @private
+     */
+    findMissingFields(expected, actual, path) {
+        const missing = [];
+        if (typeof expected === 'object' && expected !== null) {
+            if (typeof actual !== 'object' || actual === null) {
+                return [path || 'root'];
+            }
+            const expectedObj = expected;
+            const actualObj = actual;
+            for (const key in expectedObj) {
+                const fieldPath = path ? `${path}.${key}` : key;
+                if (!(key in actualObj)) {
+                    missing.push(fieldPath);
+                }
+                else {
+                    const nested = this.findMissingFields(expectedObj[key], actualObj[key], fieldPath);
+                    missing.push(...nested);
+                }
+            }
+        }
+        return missing;
+    }
+    /**
+     * Regex pattern matching.
+     * @private
+     */
+    regexMatch(expected, actual, caseSensitive) {
+        // Convert expected to regex patterns
+        const patterns = this.extractPatterns(expected);
+        const actualStr = JSON.stringify(actual);
+        const failed = [];
+        for (const pattern of patterns) {
+            const flags = caseSensitive ? '' : 'i';
+            const regex = new RegExp(pattern, flags);
+            if (!regex.test(actualStr)) {
+                failed.push(pattern);
+            }
+        }
+        if (failed.length === 0) {
+            return {
+                passed: true,
+                score: 1.0,
+                message: `All ${patterns.length} pattern(s) matched`
+            };
+        }
+        else {
+            return {
+                passed: false,
+                score: 1 - (failed.length / patterns.length),
+                message: `${failed.length} of ${patterns.length} pattern(s) failed`,
+                details: { failedPatterns: failed }
+            };
+        }
+    }
+    /**
+     * Extract regex patterns from expected output.
+     * @private
+     */
+    extractPatterns(expected) {
+        const patterns = [];
+        if (typeof expected === 'string') {
+            patterns.push(expected);
+        }
+        else if (Array.isArray(expected)) {
+            patterns.push(...expected.filter(p => typeof p === 'string'));
+        }
+        else if (typeof expected === 'object' && expected !== null) {
+            const obj = expected;
+            if (obj.responsePatterns && Array.isArray(obj.responsePatterns)) {
+                patterns.push(...obj.responsePatterns.filter(p => typeof p === 'string'));
+            }
+        }
+        return patterns;
+    }
+    /**
+     * Deep equality check.
+     * @private
+     */
+    deepEqual(expected, actual) {
+        const isEqual = this.deepEquality(expected, actual);
+        return {
+            passed: isEqual,
+            score: isEqual ? 1.0 : 0,
+            message: isEqual ? 'Output deeply equals expected' : 'Output does not deeply equal expected'
+        };
+    }
+    /**
+     * Recursive deep equality check.
+     * @private
+     */
+    deepEquality(a, b) {
+        if (a === b) {
+            return true;
+        }
+        if (a === null || b === null || a === undefined || b === undefined) {
+            return a === b;
+        }
+        if (typeof a !== typeof b) {
+            return false;
+        }
+        if (Array.isArray(a) && Array.isArray(b)) {
+            if (a.length !== b.length) {
+                return false;
+            }
+            return a.every((item, index) => this.deepEquality(item, b[index]));
+        }
+        if (typeof a === 'object' && typeof b === 'object') {
+            const aKeys = Object.keys(a);
+            const bKeys = Object.keys(b);
+            if (aKeys.length !== bKeys.length) {
+                return false;
+            }
+            const aObj = a;
+            const bObj = b;
+            return aKeys.every(key => bKeys.includes(key) && this.deepEquality(aObj[key], bObj[key]));
+        }
+        return false;
+    }
+    /**
+     * Partial match on specific fields.
+     * @private
+     */
+    partialMatch(expected, actual, fields) {
+        if (!fields || fields.length === 0) {
+            return {
+                passed: false,
+                score: 0,
+                message: 'No fields specified for partial match'
+            };
+        }
+        const failed = [];
+        for (const field of fields) {
+            const expectedValue = this.getFieldValue(expected, field);
+            const actualValue = this.getFieldValue(actual, field);
+            if (!this.deepEquality(expectedValue, actualValue)) {
+                failed.push(field);
+            }
+        }
+        if (failed.length === 0) {
+            return {
+                passed: true,
+                score: 1.0,
+                message: `All ${fields.length} field(s) matched`
+            };
+        }
+        else {
+            return {
+                passed: false,
+                score: 1 - (failed.length / fields.length),
+                message: `${failed.length} of ${fields.length} field(s) failed`,
+                details: { failedFields: failed }
+            };
+        }
+    }
+    /**
+     * Get field value by path (e.g., 'user.name' or 'items[0].id').
+     * @private
+     */
+    getFieldValue(obj, path) {
+        if (!obj || typeof obj !== 'object') {
+            return undefined;
+        }
+        const parts = path.split('.');
+        let current = obj;
+        for (const part of parts) {
+            if (!current || typeof current !== 'object') {
+                return undefined;
+            }
+            // Handle array indices
+            const arrayMatch = part.match(/^(.+)\[(\d+)\]$/);
+            if (arrayMatch) {
+                const key = arrayMatch[1];
+                const index = parseInt(arrayMatch[2]);
+                current = current[key];
+                if (!Array.isArray(current)) {
+                    return undefined;
+                }
+                current = current[index];
+            }
+            else {
+                current = current[part];
+            }
+        }
+        return current;
+    }
+}
+exports.ExactMatchOracle = ExactMatchOracle;
+//# sourceMappingURL=ExactMatchOracle.js.map

package/dist/oracles/ExactMatchOracle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ExactMatchOracle.js","sourceRoot":"","sources":["../../src/oracles/ExactMatchOracle.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAKH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,MAAa,gBAAgB;IAA7B;QACa,SAAI,GAAG,aAAa,CAAC;IAgYlC,CAAC;IA9XG;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAkB,EAAE,MAAoB;QACnD,IAAI,CAAC;YACD,MAAM,IAAI,GAAI,MAAM,CAAC,IAAe,IAAI,OAAO,CAAC;YAChD,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,KAAK,KAAK,CAAC,CAAC,eAAe;YACrE,MAAM,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,KAAK,IAAI,CAAC,CAAC,gBAAgB;YAE3E,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC;YACtC,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,CAAC;YAElC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACZ,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,6BAA6B;iBACzC,CAAC;YACN,CAAC;YAED,IAAI,CAAC,MAAM,EAAE,CAAC;gBACV,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,2BAA2B;iBACvC,CAAC;YACN,CAAC;YAED,mCAAmC;YACnC,IAAI,MAA8E,CAAC;YAEnF,QAAQ,IAAI,EAAE,CAAC;gBACX,KAAK,OAAO;oBACR,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;oBAC3C,MAAM;gBAEV,KAAK,UAAU;oBACX,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;oBAC9C,MAAM;gBAEV,KAAK,OAAO;oBACR,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC;oBAC1D,MAAM;gBAEV,KAAK,MAAM;oBACP,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;oBAC1C,MAAM;gBAEV,KAAK,SAAS;oBACV,MAAM,GAAG,IAAI,CAAC,YAAY,CACtB,QAAQ,EACR,MAAM,EACN,MAAM,CAAC,MAAkB,CAC5B,CAAC;oBACF,MAAM;gBAEV;oBACI,OAAO;wBACH,UAAU,EAAE,IAAI,CAAC,IAAI;wBACrB,MAAM,EAAE,KAAK;wBACb,KAAK,EAAE,CAAC;wBACR,OAAO,EAAE,4BAA4B,IAAI,EAAE;qBAC9C,CAAC;YACV,CAAC;YAED,OAAO;gBACH,UAAU,EAAE,IAAI,CAAC,IAAI;gBACrB,GAAG,MAAM;aACZ,CAAC;QAEN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO;gBACH,UAAU,EAAE,IAAI,CAAC,IAAI;gBACrB,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,sBAAuB,KAAe,CAAC,OAAO,EAAE;aAC5D,CAAC;QACN,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,UAAU,CACd,QAAiB,EACjB,MAAe;QAEf,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAEzC,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC5B,OAAO;gBACH,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,GAAG;gBACV,OAAO,EAAE,iCAAiC;aAC7C,CAAC;QACN,CAAC;aAAM,CAAC;YACJ,OAAO;gBACH,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,gCAAgC;gBACzC,OAAO,EAAE;oBACL,QAAQ,EAAE,WAAW,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC;oBACvC,MAAM,EAAE,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC;iBACtC;aACJ,CAAC;QACN,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,aAAa,CACjB,QAAiB,EACjB,MAAe;QAEf,MAAM,aAAa,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;QAEnE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO;gBACH,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,GAAG;gBACV,OAAO,EAAE,qCAAqC;aACjD,CAAC;QACN,CAAC;aAAM,CAAC;YACJ,OAAO;gBACH,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,WAAW,aAAa,CAAC,MAAM,oBAAoB;gBAC5D,OAAO,EAAE,EAAE,aAAa,EAAE;aAC7B,CAAC;QACN,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,iBAAiB,CACrB,QAAiB,EACjB,MAAe,EACf,IAAY;QAEZ,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YACpD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;gBAChD,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC,CAAC;YAC5B,CAAC;YAED,MAAM,WAAW,GAAG,QAAmC,CAAC;YACxD,MAAM,SAAS,GAAG,MAAiC,CAAC;YAEpD,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;gBAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;gBAEhD,IAAI,CAAC,CAAC,GAAG,IAAI,SAAS,CAAC,EAAE,CAAC;oBACtB,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC5B,CAAC;qBAAM,CAAC;oBACJ,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CACjC,WAAW,CAAC,GAAG,CAAC,EAChB,SAAS,CAAC,GAAG,CAAC,EACd,SAAS,CACZ,CAAC;oBACF,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;gBAC5B,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;OAGG;IACK,UAAU,CACd,QAAiB,EACjB,MAAe,EACf,aAAsB;QAEtB,qCAAqC;QACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAEzC,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;YACvC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YAEzC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;gBACzB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACzB,CAAC;QACL,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO;gBACH,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,GAAG;gBACV,OAAO,EAAE,OAAO,QAAQ,CAAC,MAAM,qBAAqB;aACvD,CAAC;QACN,CAAC;aAAM,CAAC;YACJ,OAAO;gBACH,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;gBAC5C,OAAO,EAAE,GAAG,MAAM,CAAC,MAAM,OAAO,QAAQ,CAAC,MAAM,oBAAoB;gBACnE,OAAO,EAAE,EAAE,cAAc,EAAE,MAAM,EAAE;aACtC,CAAC;QACN,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,eAAe,CAAC,QAAiB;QACrC,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC/B,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjC,QAAQ,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC;QAClE,CAAC;aAAM,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YAC3D,MAAM,GAAG,GAAG,QAAmC,CAAC;YAChD,IAAI,GAAG,CAAC,gBAAgB,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBAC9D,QAAQ,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC;YAC9E,CAAC;QACL,CAAC;QAED,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED;;;OAGG;IACK,SAAS,CACb,QAAiB,EACjB,MAAe;QAEf,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAEpD,OAAO;YACH,MAAM,EAAE,OAAO;YACf,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACxB,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,+BAA+B,CAAC,CAAC,CAAC,uCAAuC;SAC/F,CAAC;IACN,CAAC;IAED;;;OAGG;IACK,YAAY,CAAC,CAAU,EAAE,CAAU;QACvC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACV,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;YACjE,OAAO,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,IAAI,OAAO,CAAC,KAAK,OAAO,CAAC,EAAE,CAAC;YACxB,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YACvC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;gBACxB,OAAO,KAAK,CAAC;YACjB,CAAC;YAED,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACvE,CAAC;QAED,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAW,CAAC,CAAC;YACvC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAW,CAAC,CAAC;YAEvC,IAAI,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;gBAChC,OAAO,KAAK,CAAC;YACjB,CAAC;YAED,MAAM,IAAI,GAAG,CAA4B,CAAC;YAC1C,MAAM,IAAI,GAAG,CAA4B,CAAC;YAE1C,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CACrB,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CACjE,CAAC;QACN,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACK,YAAY,CAChB,QAAiB,EACjB,MAAe,EACf,MAAiB;QAEjB,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO;gBACH,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,uCAAuC;aACnD,CAAC;QACN,CAAC;QAED,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAC1D,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YAEtD,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,aAAa,EAAE,WAAW,CAAC,EAAE,CAAC;gBACjD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC;QACL,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO;gBACH,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,GAAG;gBACV,OAAO,EAAE,OAAO,MAAM,CAAC,MAAM,mBAAmB;aACnD,CAAC;QACN,CAAC;aAAM,CAAC;YACJ,OAAO;gBACH,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;gBAC1C,OAAO,EAAE,GAAG,MAAM,CAAC,MAAM,OAAO,MAAM,CAAC,MAAM,kBAAkB;gBAC/D,OAAO,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE;aACpC,CAAC;QACN,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,aAAa,CAAC,GAAY,EAAE,IAAY;QAC5C,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;YAClC,OAAO,SAAS,CAAC;QACrB,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,OAAO,GAAY,GAAG,CAAC;QAE3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;gBAC1C,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,uBAAuB;YACvB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YACjD,IAAI,UAAU,EAAE,CAAC;gBACb,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;gBAC1B,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;gBACtC,OAAO,GAAI,OAAmC,CAAC,GAAG,CAAC,CAAC;gBAEpD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC1B,OAAO,SAAS,CAAC;gBACrB,CAAC;gBAED,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACJ,OAAO,GAAI,OAAmC,CAAC,IAAI,CAAC,CAAC;YACzD,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;CACJ;AAjYD,4CAiYC"}

package/dist/oracles/IOracle.d.ts

@@ -0,0 +1,47 @@
+/**
+ * @fileoverview Oracle interface for test evaluation
+ * @module @memberjunction/testing-engine
+ */
+import { OracleInput, OracleConfig, OracleResult } from '../types';
+/**
+ * Interface for oracle implementations.
+ *
+ * An oracle is a source of truth that determines whether test output is correct.
+ * Oracles can be deterministic (schema validation, exact match) or heuristic (LLM judge).
+ *
+ * Each oracle type evaluates a specific aspect of test output:
+ * - SchemaValidator: Structural correctness
+ * - TraceValidator: Execution safety (no errors)
+ * - LLMJudge: Semantic quality
+ * - ExactMatch: Deterministic output comparison
+ * - SQLValidator: Database state verification
+ *
+ * @interface IOracle
+ * @example
+ * ```typescript
+ * export class SchemaValidatorOracle implements IOracle {
+ *     readonly type = 'schema-validate';
+ *
+ *     async evaluate(input: OracleInput, config: OracleConfig): Promise<OracleResult> {
+ *         // Validation logic
+ *         return { oracleType: this.type, passed: true, score: 1.0, message: 'Valid' };
+ *     }
+ * }
+ * ```
+ */
+export interface IOracle {
+    /**
+     * Unique type identifier for this oracle.
+     * Used in test configuration to specify which oracles to run.
+     */
+    readonly type: string;
+    /**
+     * Evaluate the test output.
+     *
+     * @param input - Evaluation input (test, expected/actual outputs, target entity, user)
+     * @param config - Oracle-specific configuration
+     * @returns Promise resolving to oracle result
+     */
+    evaluate(input: OracleInput, config: OracleConfig): Promise<OracleResult>;
+}
+//# sourceMappingURL=IOracle.d.ts.map

package/dist/oracles/IOracle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"IOracle.d.ts","sourceRoot":"","sources":["../../src/oracles/IOracle.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAEnE;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,WAAW,OAAO;IACpB;;;OAGG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB;;;;;;OAMG;IACH,QAAQ,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;CAC7E"}

package/dist/oracles/IOracle.js

@@ -0,0 +1,7 @@
+"use strict";
+/**
+ * @fileoverview Oracle interface for test evaluation
+ * @module @memberjunction/testing-engine
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=IOracle.js.map

package/dist/oracles/IOracle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IOracle.js","sourceRoot":"","sources":["../../src/oracles/IOracle.ts"],"names":[],"mappings":";AAAA;;;GAGG"}

package/dist/oracles/LLMJudgeOracle.d.ts

@@ -0,0 +1,65 @@
+/**
+ * @fileoverview LLM Judge oracle implementation
+ * @module @memberjunction/testing-engine
+ */
+import { IOracle } from './IOracle';
+import { OracleInput, OracleConfig, OracleResult } from '../types';
+/**
+ * LLM Judge Oracle.
+ *
+ * Uses an LLM to evaluate output quality based on custom criteria.
+ * Provides semantic evaluation beyond deterministic checks.
+ *
+ * Configuration:
+ * - criteria: Array of validation criteria (required)
+ * - model: Model to use for judging (default: from prompt or default model)
+ * - temperature: Temperature for LLM (default: 0.1 for consistency)
+ * - promptTemplate: Custom prompt template (optional, uses default if not provided)
+ * - strictMode: Require all criteria to pass (default: false, uses weighted scoring)
+ *
+ * @example
+ * ```typescript
+ * const oracle = new LLMJudgeOracle();
+ * const result = await oracle.evaluate({
+ *     actualOutput: { response: 'Sales by region report created successfully' },
+ *     expectedOutput: {
+ *         judgeValidationCriteria: [
+ *             'Response accurately answers the user\'s question',
+ *             'Response includes actionable information',
+ *             'Response is professional and clear'
+ *         ]
+ *     },
+ *     contextUser
+ * }, {
+ *     model: 'claude-sonnet-4',
+ *     temperature: 0.1
+ * });
+ * ```
+ */
+export declare class LLMJudgeOracle implements IOracle {
+    readonly type = "llm-judge";
+    /**
+     * Default prompt template for LLM judge.
+     * @private
+     */
+    private readonly defaultPromptTemplate;
+    /**
+     * Evaluate output using LLM judge.
+     *
+     * @param input - Oracle input with expected criteria and actual output
+     * @param config - Oracle configuration
+     * @returns Oracle result with LLM judgment
+     */
+    evaluate(input: OracleInput, config: OracleConfig): Promise<OracleResult>;
+    /**
+     * Build prompt for LLM judge.
+     * @private
+     */
+    private buildPrompt;
+    /**
+     * Parse LLM judgment response.
+     * @private
+     */
+    private parseJudgment;
+}
+//# sourceMappingURL=LLMJudgeOracle.d.ts.map

package/dist/oracles/LLMJudgeOracle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LLMJudgeOracle.d.ts","sourceRoot":"","sources":["../../src/oracles/LLMJudgeOracle.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAKnE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,qBAAa,cAAe,YAAW,OAAO;IAC1C,QAAQ,CAAC,IAAI,eAAe;IAE5B;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CA4BvC;IAEC;;;;;;OAMG;IACG,QAAQ,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IA+F/E;;;OAGG;IACH,OAAO,CAAC,WAAW;IAqBnB;;;OAGG;IACH,OAAO,CAAC,aAAa;CA6BxB"}