@memberjunction/testing-engine 0.0.1 → 2.119.0

package/dist/oracles/LLMJudgeOracle.js

@@ -0,0 +1,214 @@
+"use strict";
+/**
+ * @fileoverview LLM Judge oracle implementation
+ * @module @memberjunction/testing-engine
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LLMJudgeOracle = void 0;
+const ai_core_plus_1 = require("@memberjunction/ai-core-plus");
+const ai_prompts_1 = require("@memberjunction/ai-prompts");
+const aiengine_1 = require("@memberjunction/aiengine");
+/**
+ * LLM Judge Oracle.
+ *
+ * Uses an LLM to evaluate output quality based on custom criteria.
+ * Provides semantic evaluation beyond deterministic checks.
+ *
+ * Configuration:
+ * - criteria: Array of validation criteria (required)
+ * - model: Model to use for judging (default: from prompt or default model)
+ * - temperature: Temperature for LLM (default: 0.1 for consistency)
+ * - promptTemplate: Custom prompt template (optional, uses default if not provided)
+ * - strictMode: Require all criteria to pass (default: false, uses weighted scoring)
+ *
+ * @example
+ * ```typescript
+ * const oracle = new LLMJudgeOracle();
+ * const result = await oracle.evaluate({
+ *     actualOutput: { response: 'Sales by region report created successfully' },
+ *     expectedOutput: {
+ *         judgeValidationCriteria: [
+ *             'Response accurately answers the user\'s question',
+ *             'Response includes actionable information',
+ *             'Response is professional and clear'
+ *         ]
+ *     },
+ *     contextUser
+ * }, {
+ *     model: 'claude-sonnet-4',
+ *     temperature: 0.1
+ * });
+ * ```
+ */
+class LLMJudgeOracle {
+    constructor() {
+        this.type = 'llm-judge';
+        /**
+         * Default prompt template for LLM judge.
+         * @private
+         */
+        this.defaultPromptTemplate = `You are an expert AI evaluator. Your task is to judge whether an AI agent's output meets the specified validation criteria.
+
+**Input:**
+{{input}}
+
+**Expected Output Requirements:**
+{{expected}}
+
+**Actual Output:**
+{{actual}}
+
+**Validation Criteria:**
+{{criteria}}
+
+**Instructions:**
+For each criterion, evaluate whether the actual output satisfies it. Provide:
+1. A score from 0.0 to 1.0 for each criterion
+2. A brief explanation for each score
+3. An overall assessment
+
+Respond in JSON format:
+{
+  "criteriaScores": [
+    { "criterion": "...", "score": 0.0-1.0, "explanation": "..." }
+  ],
+  "overallScore": 0.0-1.0,
+  "overallAssessment": "...",
+  "passed": true/false
+}`;
+    }
+    /**
+     * Evaluate output using LLM judge.
+     *
+     * @param input - Oracle input with expected criteria and actual output
+     * @param config - Oracle configuration
+     * @returns Oracle result with LLM judgment
+     */
+    async evaluate(input, config) {
+        try {
+            // Ensure AIEngine is configured
+            await aiengine_1.AIEngine.Instance.Config(false, input.contextUser);
+            // Get criteria from expected outcomes or config
+            const criteria = input.expectedOutput?.judgeValidationCriteria ||
+                config.criteria;
+            if (!criteria || criteria.length === 0) {
+                return {
+                    oracleType: this.type,
+                    passed: false,
+                    score: 0,
+                    message: 'No validation criteria provided'
+                };
+            }
+            // Find the LLM Judge prompt from AIEngine
+            const judgePrompt = aiengine_1.AIEngine.Instance.Prompts.find(p => p.Name === 'Test LLM Judge' || p.Name.toLowerCase().includes('llm judge'));
+            if (!judgePrompt) {
+                return {
+                    oracleType: this.type,
+                    passed: false,
+                    score: 0,
+                    message: 'LLM Judge prompt not found in AIEngine.Instance.Prompts. Please create a prompt named "Test LLM Judge".'
+                };
+            }
+            // Prepare data for prompt template
+            const inputDefinition = input.test.InputDefinition ?
+                (typeof input.test.InputDefinition === 'string' ?
+                    JSON.parse(input.test.InputDefinition) :
+                    input.test.InputDefinition) :
+                {};
+            const promptData = {
+                input: JSON.stringify(inputDefinition, null, 2),
+                expected: JSON.stringify(input.expectedOutput, null, 2),
+                actual: JSON.stringify(input.actualOutput, null, 2),
+                criteria: criteria.map((c, i) => `${i + 1}. ${c}`).join('\n')
+            };
+            // Execute LLM judgment
+            const promptParams = new ai_core_plus_1.AIPromptParams();
+            promptParams.prompt = judgePrompt;
+            promptParams.data = promptData;
+            promptParams.contextUser = input.contextUser;
+            const runner = new ai_prompts_1.AIPromptRunner();
+            const result = await runner.ExecutePrompt(promptParams);
+            if (!result.success) {
+                return {
+                    oracleType: this.type,
+                    passed: false,
+                    score: 0,
+                    message: `LLM judgment failed: ${result.errorMessage}`
+                };
+            }
+            // Parse LLM response
+            const judgment = this.parseJudgment(result.result);
+            // Determine pass/fail
+            const strictMode = config.strictMode;
+            const passed = strictMode
+                ? judgment.criteriaScores.every(s => s.score >= 0.8)
+                : judgment.overallScore >= 0.7;
+            return {
+                oracleType: this.type,
+                passed,
+                score: judgment.overallScore,
+                message: judgment.overallAssessment,
+                details: {
+                    criteriaScores: judgment.criteriaScores,
+                    llmModel: config.model || 'default',
+                    llmCost: result.cost
+                }
+            };
+        }
+        catch (error) {
+            return {
+                oracleType: this.type,
+                passed: false,
+                score: 0,
+                message: `LLM judge error: ${error.message}`
+            };
+        }
+    }
+    /**
+     * Build prompt for LLM judge.
+     * @private
+     */
+    buildPrompt(input, criteria, customTemplate) {
+        const template = customTemplate || this.defaultPromptTemplate;
+        // Format input data
+        const inputStr = JSON.stringify(input.test.InputDefinition, null, 2);
+        const expectedStr = JSON.stringify(input.expectedOutput, null, 2);
+        const actualStr = JSON.stringify(input.actualOutput, null, 2);
+        const criteriaStr = criteria.map((c, i) => `${i + 1}. ${c}`).join('\n');
+        // Replace placeholders
+        return template
+            .replace('{{input}}', inputStr)
+            .replace('{{expected}}', expectedStr)
+            .replace('{{actual}}', actualStr)
+            .replace('{{criteria}}', criteriaStr);
+    }
+    /**
+     * Parse LLM judgment response.
+     * @private
+     */
+    parseJudgment(response) {
+        try {
+            // Try to extract JSON from response
+            const jsonMatch = response.match(/\{[\s\S]*\}/);
+            if (!jsonMatch) {
+                throw new Error('No JSON found in LLM response');
+            }
+            const parsed = JSON.parse(jsonMatch[0]);
+            return {
+                criteriaScores: parsed.criteriaScores || [],
+                overallScore: parsed.overallScore || 0,
+                overallAssessment: parsed.overallAssessment || 'No assessment provided'
+            };
+        }
+        catch (error) {
+            // Fallback to simple parsing
+            return {
+                criteriaScores: [],
+                overallScore: 0,
+                overallAssessment: `Failed to parse LLM response: ${response.substring(0, 200)}`
+            };
+        }
+    }
+}
+exports.LLMJudgeOracle = LLMJudgeOracle;
+//# sourceMappingURL=LLMJudgeOracle.js.map

package/dist/oracles/LLMJudgeOracle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LLMJudgeOracle.js","sourceRoot":"","sources":["../../src/oracles/LLMJudgeOracle.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAIH,+DAA8D;AAC9D,2DAA4D;AAC5D,uDAAoD;AAEpD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,MAAa,cAAc;IAA3B;QACa,SAAI,GAAG,WAAW,CAAC;QAE5B;;;WAGG;QACc,0BAAqB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA4B3C,CAAC;IAkKH,CAAC;IAhKG;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAkB,EAAE,MAAoB;QACnD,IAAI,CAAC;YACD,gCAAgC;YAChC,MAAM,mBAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;YAEzD,gDAAgD;YAChD,MAAM,QAAQ,GAAK,KAAK,CAAC,cAAsB,EAAE,uBAAoC;gBACrE,MAAM,CAAC,QAAqB,CAAC;YAE7C,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrC,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,iCAAiC;iBAC7C,CAAC;YACN,CAAC;YAED,0CAA0C;YAC1C,MAAM,WAAW,GAAG,mBAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CACnD,CAAC,CAAC,IAAI,KAAK,gBAAgB,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAC5E,CAAC;YAEF,IAAI,CAAC,WAAW,EAAE,CAAC;gBACf,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,yGAAyG;iBACrH,CAAC;YACN,CAAC;YAED,mCAAmC;YACnC,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;gBAChD,CAAC,OAAO,KAAK,CAAC,IAAI,CAAC,eAAe,KAAK,QAAQ,CAAC,CAAC;oBAC7C,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC;oBACxC,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC;gBACjC,EAAE,CAAC;YAEP,MAAM,UAAU,GAAG;gBACf,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC/C,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;gBACvD,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;gBACnD,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;aAChE,CAAC;YAEF,uBAAuB;YACvB,MAAM,YAAY,GAAG,IAAI,6BAAc,EAAE,CAAC;YAC1C,YAAY,CAAC,MAAM,GAAG,WAAW,CAAC;YAClC,YAAY,CAAC,IAAI,GAAG,UAAU,CAAC;YAC/B,YAAY,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC;YAE7C,MAAM,MAAM,GAAG,IAAI,2BAAc,EAAE,CAAC;YACpC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC;YAExD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBAClB,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,wBAAwB,MAAM,CAAC,YAAY,EAAE;iBACzD,CAAC;YACN,CAAC;YAED,qBAAqB;YACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,MAAgB,CAAC,CAAC;YAE7D,sBAAsB;YACtB,MAAM,UAAU,GAAG,MAAM,CAAC,UAAqB,CAAC;YAChD,MAAM,MAAM,GAAG,UAAU;gBACrB,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,CAAC;gBACpD,CAAC,CAAC,QAAQ,CAAC,YAAY,IAAI,GAAG,CAAC;YAEnC,OAAO;gBACH,UAAU,EAAE,IAAI,CAAC,IAAI;gBACrB,MAAM;gBACN,KAAK,EAAE,QAAQ,CAAC,YAAY;gBAC5B,OAAO,EAAE,QAAQ,CAAC,iBAAiB;gBACnC,OAAO,EAAE;oBACL,cAAc,EAAE,QAAQ,CAAC,cAAc;oBACvC,QAAQ,EAAE,MAAM,CAAC,KAAK,IAAI,SAAS;oBACnC,OAAO,EAAE,MAAM,CAAC,IAAI;iBACvB;aACJ,CAAC;QAEN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO;gBACH,UAAU,EAAE,IAAI,CAAC,IAAI;gBACrB,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,oBAAqB,KAAe,CAAC,OAAO,EAAE;aAC1D,CAAC;QACN,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,WAAW,CACf,KAAkB,EAClB,QAAkB,EAClB,cAAuB;QAEvB,MAAM,QAAQ,GAAG,cAAc,IAAI,IAAI,CAAC,qBAAqB,CAAC;QAE9D,oBAAoB;QACpB,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACrE,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAClE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAC9D,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAExE,uBAAuB;QACvB,OAAO,QAAQ;aACV,OAAO,CAAC,WAAW,EAAE,QAAQ,CAAC;aAC9B,OAAO,CAAC,cAAc,EAAE,WAAW,CAAC;aACpC,OAAO,CAAC,YAAY,EAAE,SAAS,CAAC;aAChC,OAAO,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;IAC9C,CAAC;IAED;;;OAGG;IACK,aAAa,CAAC,QAAgB;QAKlC,IAAI,CAAC;YACD,oCAAoC;YACpC,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAChD,IAAI,CAAC,SAAS,EAAE,CAAC;gBACb,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;YACrD,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;YAExC,OAAO;gBACH,cAAc,EAAE,MAAM,CAAC,cAAc,IAAI,EAAE;gBAC3C,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,CAAC;gBACtC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB,IAAI,wBAAwB;aAC1E,CAAC;QAEN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,6BAA6B;YAC7B,OAAO;gBACH,cAAc,EAAE,EAAE;gBAClB,YAAY,EAAE,CAAC;gBACf,iBAAiB,EAAE,iCAAiC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;aACnF,CAAC;QACN,CAAC;IACL,CAAC;CACJ;AArMD,wCAqMC"}

package/dist/oracles/SQLValidatorOracle.d.ts

@@ -0,0 +1,78 @@
+/**
+ * @fileoverview SQL validation oracle implementation
+ * @module @memberjunction/testing-engine
+ */
+import { IOracle } from './IOracle';
+import { OracleInput, OracleConfig, OracleResult } from '../types';
+/**
+ * SQL Validator Oracle.
+ *
+ * Validates database state by executing SQL queries and comparing results.
+ * Useful for testing that agent actions had the expected database effects.
+ *
+ * Configuration:
+ * - queries: Array of SQL validation queries
+ * - requireAll: Whether all queries must pass (default: true)
+ *
+ * Each query object contains:
+ * - sql: The SQL query to execute
+ * - expectedResult: Expected result (can be value, row count, or boolean)
+ * - description: Human-readable description of what the query validates
+ *
+ * @example
+ * ```typescript
+ * const oracle = new SQLValidatorOracle();
+ * const result = await oracle.evaluate({
+ *     expectedOutput: {
+ *         sqlValidations: [
+ *             {
+ *                 sql: "SELECT COUNT(*) FROM Reports WHERE Name LIKE '%Sales%'",
+ *                 expectedResult: { count: 1 },
+ *                 description: "Sales report was created"
+ *             },
+ *             {
+ *                 sql: "SELECT Status FROM Reports WHERE ID = @ReportID",
+ *                 expectedResult: { status: 'Published' },
+ *                 description: "Report status is Published"
+ *             }
+ *         ]
+ *     },
+ *     actualOutput: {
+ *         reportId: 'abc-123'
+ *     },
+ *     contextUser
+ * }, {});
+ * ```
+ */
+export declare class SQLValidatorOracle implements IOracle {
+    readonly type = "sql-validate";
+    /**
+     * Evaluate database state using SQL queries.
+     *
+     * @param input - Oracle input with SQL validations and actual output
+     * @param config - Oracle configuration
+     * @returns Oracle result with query validation details
+     */
+    evaluate(input: OracleInput, config: OracleConfig): Promise<OracleResult>;
+    /**
+     * Execute a single validation query.
+     * @private
+     */
+    private executeValidation;
+    /**
+     * Replace parameters in SQL query with actual values.
+     * @private
+     */
+    private replaceParameters;
+    /**
+     * Extract result from query result set.
+     * @private
+     */
+    private extractResult;
+    /**
+     * Compare expected and actual results.
+     * @private
+     */
+    private compareResults;
+}
+//# sourceMappingURL=SQLValidatorOracle.d.ts.map

package/dist/oracles/SQLValidatorOracle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"SQLValidatorOracle.d.ts","sourceRoot":"","sources":["../../src/oracles/SQLValidatorOracle.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAEnE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,qBAAa,kBAAmB,YAAW,OAAO;IAC9C,QAAQ,CAAC,IAAI,kBAAkB;IAE/B;;;;;;OAMG;IACG,QAAQ,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAgE/E;;;OAGG;YACW,iBAAiB;IAqD/B;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IA8BzB;;;OAGG;IACH,OAAO,CAAC,aAAa;IAqBrB;;;OAGG;IACH,OAAO,CAAC,cAAc;CA4BzB"}

package/dist/oracles/SQLValidatorOracle.js

@@ -0,0 +1,215 @@
+"use strict";
+/**
+ * @fileoverview SQL validation oracle implementation
+ * @module @memberjunction/testing-engine
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SQLValidatorOracle = void 0;
+const core_1 = require("@memberjunction/core");
+/**
+ * SQL Validator Oracle.
+ *
+ * Validates database state by executing SQL queries and comparing results.
+ * Useful for testing that agent actions had the expected database effects.
+ *
+ * Configuration:
+ * - queries: Array of SQL validation queries
+ * - requireAll: Whether all queries must pass (default: true)
+ *
+ * Each query object contains:
+ * - sql: The SQL query to execute
+ * - expectedResult: Expected result (can be value, row count, or boolean)
+ * - description: Human-readable description of what the query validates
+ *
+ * @example
+ * ```typescript
+ * const oracle = new SQLValidatorOracle();
+ * const result = await oracle.evaluate({
+ *     expectedOutput: {
+ *         sqlValidations: [
+ *             {
+ *                 sql: "SELECT COUNT(*) FROM Reports WHERE Name LIKE '%Sales%'",
+ *                 expectedResult: { count: 1 },
+ *                 description: "Sales report was created"
+ *             },
+ *             {
+ *                 sql: "SELECT Status FROM Reports WHERE ID = @ReportID",
+ *                 expectedResult: { status: 'Published' },
+ *                 description: "Report status is Published"
+ *             }
+ *         ]
+ *     },
+ *     actualOutput: {
+ *         reportId: 'abc-123'
+ *     },
+ *     contextUser
+ * }, {});
+ * ```
+ */
+class SQLValidatorOracle {
+    constructor() {
+        this.type = 'sql-validate';
+    }
+    /**
+     * Evaluate database state using SQL queries.
+     *
+     * @param input - Oracle input with SQL validations and actual output
+     * @param config - Oracle configuration
+     * @returns Oracle result with query validation details
+     */
+    async evaluate(input, config) {
+        try {
+            // Get SQL validations from expected outcomes
+            const validations = input.expectedOutput?.sqlValidations || [];
+            if (validations.length === 0) {
+                return {
+                    oracleType: this.type,
+                    passed: false,
+                    score: 0,
+                    message: 'No SQL validations provided in ExpectedOutcomes.sqlValidations'
+                };
+            }
+            const requireAll = config.requireAll !== false; // Default true
+            const results = [];
+            // Execute each validation query
+            for (const validation of validations) {
+                const queryResult = await this.executeValidation(validation, input.actualOutput, input.contextUser);
+                results.push(queryResult);
+            }
+            // Calculate score and determine pass/fail
+            const passedCount = results.filter(r => r.passed).length;
+            const totalCount = results.length;
+            const score = totalCount > 0 ? passedCount / totalCount : 0;
+            const passed = requireAll ? passedCount === totalCount : passedCount > 0;
+            return {
+                oracleType: this.type,
+                passed,
+                score,
+                message: requireAll
+                    ? `${passedCount}/${totalCount} validation(s) passed`
+                    : `At least one validation passed (${passedCount}/${totalCount})`,
+                details: { validations: results }
+            };
+        }
+        catch (error) {
+            return {
+                oracleType: this.type,
+                passed: false,
+                score: 0,
+                message: `SQL validation error: ${error.message}`
+            };
+        }
+    }
+    /**
+     * Execute a single validation query.
+     * @private
+     */
+    async executeValidation(validation, actualOutput, contextUser) {
+        try {
+            // Replace parameters in SQL with values from actualOutput
+            const sql = this.replaceParameters(validation.sql, actualOutput);
+            // Get database provider from Metadata.Provider
+            const dbProvider = core_1.Metadata.Provider;
+            // Execute the SQL query
+            const queryResults = await dbProvider.ExecuteSQL(sql, undefined, // No parameters (already substituted in SQL string)
+            { description: validation.description || 'SQL Validation' }, contextUser);
+            // Extract result (handle single value, single row, or multiple rows)
+            const actualResult = this.extractResult(queryResults);
+            // Compare with expected result
+            const passed = this.compareResults(validation.expectedResult, actualResult);
+            return {
+                description: validation.description || 'SQL validation',
+                sql: validation.sql,
+                passed,
+                expected: validation.expectedResult,
+                actual: actualResult
+            };
+        }
+        catch (error) {
+            return {
+                description: validation.description || 'SQL validation',
+                sql: validation.sql,
+                passed: false,
+                expected: validation.expectedResult,
+                actual: null,
+                error: error.message
+            };
+        }
+    }
+    /**
+     * Replace parameters in SQL query with actual values.
+     * @private
+     */
+    replaceParameters(sql, actualOutput) {
+        if (!actualOutput || typeof actualOutput !== 'object') {
+            return sql;
+        }
+        let result = sql;
+        const outputObj = actualOutput;
+        // Replace @ParameterName with values from actualOutput
+        const paramMatches = sql.matchAll(/@(\w+)/g);
+        for (const match of paramMatches) {
+            const paramName = match[1];
+            const camelCaseParam = paramName.charAt(0).toLowerCase() + paramName.slice(1);
+            // Try both original and camelCase versions
+            const value = outputObj[paramName] || outputObj[camelCaseParam];
+            if (value !== undefined) {
+                // Properly escape and quote the value
+                const escapedValue = typeof value === 'string'
+                    ? `'${value.replace(/'/g, "''")}'`
+                    : String(value);
+                result = result.replace(new RegExp(`@${paramName}`, 'g'), escapedValue);
+            }
+        }
+        return result;
+    }
+    /**
+     * Extract result from query result set.
+     * @private
+     */
+    extractResult(results) {
+        if (!results || results.length === 0) {
+            return null;
+        }
+        // If single row, single column, return the value
+        if (results.length === 1) {
+            const row = results[0];
+            const keys = Object.keys(row);
+            if (keys.length === 1) {
+                return row[keys[0]];
+            }
+            return row;
+        }
+        // Multiple rows
+        return results;
+    }
+    /**
+     * Compare expected and actual results.
+     * @private
+     */
+    compareResults(expected, actual) {
+        // Handle null/undefined
+        if (expected === null || expected === undefined) {
+            return actual === null || actual === undefined;
+        }
+        // Handle boolean
+        if (typeof expected === 'boolean') {
+            return Boolean(actual) === expected;
+        }
+        // Handle number
+        if (typeof expected === 'number') {
+            return Number(actual) === expected;
+        }
+        // Handle string
+        if (typeof expected === 'string') {
+            return String(actual) === expected;
+        }
+        // Handle object/array
+        if (typeof expected === 'object') {
+            return JSON.stringify(expected) === JSON.stringify(actual);
+        }
+        return false;
+    }
+}
+exports.SQLValidatorOracle = SQLValidatorOracle;
+//# sourceMappingURL=SQLValidatorOracle.js.map

package/dist/oracles/SQLValidatorOracle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SQLValidatorOracle.js","sourceRoot":"","sources":["../../src/oracles/SQLValidatorOracle.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,+CAAgF;AAIhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,MAAa,kBAAkB;IAA/B;QACa,SAAI,GAAG,cAAc,CAAC;IA6NnC,CAAC;IA3NG;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAkB,EAAE,MAAoB;QACnD,IAAI,CAAC;YACD,6CAA6C;YAC7C,MAAM,WAAW,GAAK,KAAK,CAAC,cAAsB,EAAE,cAIjD,IAAI,EAAE,CAAC;YAEV,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3B,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,gEAAgE;iBAC5E,CAAC;YACN,CAAC;YAED,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,KAAK,KAAK,CAAC,CAAC,eAAe;YAC/D,MAAM,OAAO,GAOR,EAAE,CAAC;YAER,gCAAgC;YAChC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACnC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAC5C,UAAU,EACV,KAAK,CAAC,YAAY,EAClB,KAAK,CAAC,WAAW,CACpB,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC9B,CAAC;YAED,0CAA0C;YAC1C,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;YACzD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;YAClC,MAAM,KAAK,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5D,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,WAAW,KAAK,UAAU,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC;YAEzE,OAAO;gBACH,UAAU,EAAE,IAAI,CAAC,IAAI;gBACrB,MAAM;gBACN,KAAK;gBACL,OAAO,EAAE,UAAU;oBACf,CAAC,CAAC,GAAG,WAAW,IAAI,UAAU,uBAAuB;oBACrD,CAAC,CAAC,mCAAmC,WAAW,IAAI,UAAU,GAAG;gBACrE,OAAO,EAAE,EAAE,WAAW,EAAE,OAAO,EAAE;aACpC,CAAC;QAEN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO;gBACH,UAAU,EAAE,IAAI,CAAC,IAAI;gBACrB,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,yBAA0B,KAAe,CAAC,OAAO,EAAE;aAC/D,CAAC;QACN,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,iBAAiB,CAC3B,UAA0E,EAC1E,YAAqB,EACrB,WAAqB;QASrB,IAAI,CAAC;YACD,0DAA0D;YAC1D,MAAM,GAAG,GAAG,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;YAEjE,+CAA+C;YAC/C,MAAM,UAAU,GAAG,eAAQ,CAAC,QAAgC,CAAC;YAE7D,wBAAwB;YACxB,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,UAAU,CAC5C,GAAG,EACH,SAAS,EAAG,oDAAoD;YAChE,EAAE,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,gBAAgB,EAAE,EAC3D,WAAW,CACd,CAAC;YAEF,qEAAqE;YACrE,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC;YAEtD,+BAA+B;YAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;YAE5E,OAAO;gBACH,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,gBAAgB;gBACvD,GAAG,EAAE,UAAU,CAAC,GAAG;gBACnB,MAAM;gBACN,QAAQ,EAAE,UAAU,CAAC,cAAc;gBACnC,MAAM,EAAE,YAAY;aACvB,CAAC;QAEN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO;gBACH,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,gBAAgB;gBACvD,GAAG,EAAE,UAAU,CAAC,GAAG;gBACnB,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,UAAU,CAAC,cAAc;gBACnC,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAG,KAAe,CAAC,OAAO;aAClC,CAAC;QACN,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,iBAAiB,CAAC,GAAW,EAAE,YAAqB;QACxD,IAAI,CAAC,YAAY,IAAI,OAAO,YAAY,KAAK,QAAQ,EAAE,CAAC;YACpD,OAAO,GAAG,CAAC;QACf,CAAC;QAED,IAAI,MAAM,GAAG,GAAG,CAAC;QACjB,MAAM,SAAS,GAAG,YAAuC,CAAC;QAE1D,uDAAuD;QACvD,MAAM,YAAY,GAAG,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC7C,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;YAC/B,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAC3B,MAAM,cAAc,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAE9E,2CAA2C;YAC3C,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC,cAAc,CAAC,CAAC;YAEhE,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACtB,sCAAsC;gBACtC,MAAM,YAAY,GAAG,OAAO,KAAK,KAAK,QAAQ;oBAC1C,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG;oBAClC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAEpB,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,IAAI,SAAS,EAAE,EAAE,GAAG,CAAC,EAAE,YAAY,CAAC,CAAC;YAC5E,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;OAGG;IACK,aAAa,CAAC,OAAkB;QACpC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,iDAAiD;QACjD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAA4B,CAAC;YAClD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAE9B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACpB,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACxB,CAAC;YAED,OAAO,GAAG,CAAC;QACf,CAAC;QAED,gBAAgB;QAChB,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;OAGG;IACK,cAAc,CAAC,QAAiB,EAAE,MAAe;QACrD,wBAAwB;QACxB,IAAI,QAAQ,KAAK,IAAI,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC9C,OAAO,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,SAAS,CAAC;QACnD,CAAC;QAED,iBAAiB;QACjB,IAAI,OAAO,QAAQ,KAAK,SAAS,EAAE,CAAC;YAChC,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC;QACxC,CAAC;QAED,gBAAgB;QAChB,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC/B,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC;QACvC,CAAC;QAED,gBAAgB;QAChB,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC/B,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC;QACvC,CAAC;QAED,sBAAsB;QACtB,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC/B,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC/D,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;CACJ;AA9ND,gDA8NC"}

package/dist/oracles/SchemaValidatorOracle.d.ts

@@ -0,0 +1,61 @@
+/**
+ * @fileoverview Schema validation oracle implementation
+ * @module @memberjunction/testing-engine
+ */
+import { IOracle } from './IOracle';
+import { OracleInput, OracleConfig, OracleResult } from '../types';
+/**
+ * Schema Validator Oracle.
+ *
+ * Validates that actual output conforms to an expected JSON schema.
+ * Uses JSON Schema draft-07 specification for validation.
+ *
+ * Configuration:
+ * - schema: JSON Schema object defining expected structure
+ * - strict: Whether to fail on additional properties (default: false)
+ *
+ * @example
+ * ```typescript
+ * const oracle = new SchemaValidatorOracle();
+ * const result = await oracle.evaluate({
+ *     actualOutput: { name: 'John', age: 30 },
+ *     expectedOutput: {
+ *         responseSchema: {
+ *             type: 'object',
+ *             required: ['name', 'age'],
+ *             properties: {
+ *                 name: { type: 'string' },
+ *                 age: { type: 'number' }
+ *             }
+ *         }
+ *     }
+ * }, {});
+ * ```
+ */
+export declare class SchemaValidatorOracle implements IOracle {
+    readonly type = "schema-validate";
+    /**
+     * Evaluate actual output against JSON schema.
+     *
+     * @param input - Oracle input with expected schema and actual output
+     * @param config - Oracle configuration
+     * @returns Oracle result with pass/fail and validation details
+     */
+    evaluate(input: OracleInput, config: OracleConfig): Promise<OracleResult>;
+    /**
+     * Validate data against JSON schema.
+     * @private
+     */
+    private validateAgainstSchema;
+    /**
+     * Validate a single value against schema.
+     * @private
+     */
+    private validateValue;
+    /**
+     * Get JSON Schema type for a value.
+     * @private
+     */
+    private getType;
+}
+//# sourceMappingURL=SchemaValidatorOracle.d.ts.map

package/dist/oracles/SchemaValidatorOracle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"SchemaValidatorOracle.d.ts","sourceRoot":"","sources":["../../src/oracles/SchemaValidatorOracle.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAEnE;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,qBAAsB,YAAW,OAAO;IACjD,QAAQ,CAAC,IAAI,qBAAqB;IAElC;;;;;;OAMG;IACG,QAAQ,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IA+C/E;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAc7B;;;OAGG;IACH,OAAO,CAAC,aAAa;IAiFrB;;;OAGG;IACH,OAAO,CAAC,OAAO;CA6BlB"}