@memberjunction/testing-engine 0.0.1 → 2.119.0

package/dist/drivers/AgentEvalDriver.js

@@ -0,0 +1,370 @@
+"use strict";
+/**
+ * @fileoverview Test driver for AI Agent evaluation
+ * @module @memberjunction/testing-engine
+ */
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AgentEvalDriver = void 0;
+const global_1 = require("@memberjunction/global");
+const ai_agents_1 = require("@memberjunction/ai-agents");
+const BaseTestDriver_1 = require("./BaseTestDriver");
+/**
+ * Test driver for AI Agent evaluation.
+ *
+ * Executes an AI agent with test input, runs configured oracles,
+ * and creates bidirectional link between TestRun and AgentRun.
+ *
+ * @example
+ * ```typescript
+ * // Configuration JSON in Test entity
+ * {
+ *   "agentId": "agent-123",
+ *   "oracles": [
+ *     { "type": "trace-no-errors", "weight": 0.2 },
+ *     { "type": "llm-judge", "weight": 0.5, "config": { "criteria": [...] } },
+ *     { "type": "schema-validate", "weight": 0.3, "config": { "schema": {...} } }
+ *   ],
+ *   "scoringWeights": { "trace-no-errors": 0.2, "llm-judge": 0.5, "schema-validate": 0.3 }
+ * }
+ *
+ * // InputDefinition JSON in Test entity
+ * {
+ *   "userMessage": "Create a report showing sales by region",
+ *   "conversationContext": null,
+ *   "executionParams": { "temperatureOverride": 0.3 }
+ * }
+ *
+ * // ExpectedOutcomes JSON in Test entity
+ * {
+ *   "responsePatterns": ["sales.*region", "chart|graph"],
+ *   "expectedEntities": ["Report", "Dashboard"],
+ *   "responseSchema": { "type": "object", "properties": {...} },
+ *   "judgeValidationCriteria": [
+ *     "Response accurately answers the user's question",
+ *     "Report includes proper data visualization",
+ *     "Response is professional and clear"
+ *   ]
+ * }
+ * ```
+ */
+let AgentEvalDriver = class AgentEvalDriver extends BaseTestDriver_1.BaseTestDriver {
+    /**
+     * Execute agent evaluation test.
+     *
+     * Steps:
+     * 1. Parse configuration and input
+     * 2. Load and execute agent via AgentRunner
+     * 3. Create bidirectional link (TestRun ↔ AgentRun)
+     * 4. Run oracles to evaluate results
+     * 5. Calculate score and determine status
+     * 6. Return structured results
+     *
+     * @param context - Execution context
+     * @returns Execution result
+     */
+    async Execute(context) {
+        this.log('Starting agent evaluation', context.options.verbose);
+        try {
+            // Parse configuration
+            const config = this.parseConfig(context.test);
+            const input = this.parseInputDefinition(context.test);
+            const expected = this.parseExpectedOutcomes(context.test);
+            // Load agent
+            const agent = await this.loadAgent(config.agentId, context.contextUser);
+            // Execute agent
+            this.log(`Executing agent: ${agent.Name}`, context.options.verbose);
+            const agentResult = await this.executeAgent(agent, input, context.contextUser, context.test, config.maxExecutionTime, context.testRun);
+            const agentRun = agentResult.agentRun;
+            // Create bidirectional link
+            await this.linkTestRunToAgentRun(context.testRun, agentRun);
+            // Extract actual output
+            const actualOutput = this.extractAgentOutput(agentRun);
+            // Run oracles
+            this.log('Running oracles for evaluation', context.options.verbose);
+            const oracleResults = await this.runOracles(config, input, expected, actualOutput, agentRun, context);
+            // Calculate score and status
+            // When oracles are disabled, consider test passed if agent succeeded
+            const score = this.calculateScore(oracleResults, config.scoringWeights);
+            const status = oracleResults.length === 0 && agentRun.Status === 'Completed'
+                ? 'Passed'
+                : this.determineStatus(oracleResults);
+            // Count checks
+            const passedChecks = oracleResults.filter(r => r.passed).length;
+            const totalChecks = oracleResults.length;
+            // Calculate cost
+            const totalCost = this.calculateTotalCost(agentRun);
+            // Calculate duration in MS
+            const durationMs = this.calculateDurationMs(agentRun);
+            // Build result
+            const result = {
+                targetType: 'AI Agent',
+                targetLogId: agentRun.ID,
+                status,
+                score,
+                oracleResults,
+                passedChecks,
+                failedChecks: totalChecks - passedChecks,
+                totalChecks,
+                inputData: input,
+                expectedOutput: expected,
+                actualOutput,
+                totalCost,
+                durationMs
+            };
+            this.log(`Agent evaluation completed: ${status} (Score: ${score})`, context.options.verbose);
+            return result;
+        }
+        catch (error) {
+            this.logError('Agent evaluation failed', error);
+            throw error;
+        }
+    }
+    /**
+     * Validate agent evaluation test configuration.
+     *
+     * Checks:
+     * - Base validation (InputDefinition, ExpectedOutcomes, Configuration)
+     * - Agent ID is valid
+     * - At least one oracle is configured
+     * - Oracle types are registered
+     * - Scoring weights are valid
+     *
+     * @param test - Test entity to validate
+     * @returns Validation result
+     */
+    async Validate(test) {
+        // Run base validation
+        const baseResult = await super.Validate(test);
+        if (!baseResult.valid) {
+            return baseResult;
+        }
+        const errors = [...baseResult.errors];
+        const warnings = [...baseResult.warnings];
+        try {
+            // Parse and validate configuration
+            const config = this.parseConfig(test);
+            // Validate agent ID
+            if (!config.agentId) {
+                errors.push({
+                    category: 'configuration',
+                    message: 'agentId is required in Configuration',
+                    field: 'Configuration.agentId',
+                    suggestion: 'Specify the ID of the agent to test'
+                });
+            }
+            // Note: We cannot validate agent existence without contextUser
+            // That validation will happen at execution time
+            // Validate oracles configuration exists
+            if (!config.oracles || config.oracles.length === 0) {
+                errors.push({
+                    category: 'configuration',
+                    message: 'At least one oracle is required',
+                    field: 'Configuration.oracles',
+                    suggestion: 'Add oracle configurations (e.g., trace-no-errors, llm-judge)'
+                });
+            }
+            // Note: Oracle type validation requires registry from execution context
+            // That validation will happen at execution time
+            // Validate scoring weights
+            if (config.scoringWeights) {
+                const totalWeight = Object.values(config.scoringWeights).reduce((sum, w) => sum + w, 0);
+                if (Math.abs(totalWeight - 1.0) > 0.01) {
+                    warnings.push({
+                        category: 'best-practice',
+                        message: 'Scoring weights should sum to 1.0',
+                        recommendation: `Current sum: ${totalWeight.toFixed(2)}`
+                    });
+                }
+            }
+            // Validate input definition
+            const input = this.parseInputDefinition(test);
+            if (!input.userMessage || input.userMessage.trim() === '') {
+                errors.push({
+                    category: 'input',
+                    message: 'userMessage is required in InputDefinition',
+                    field: 'InputDefinition.userMessage',
+                    suggestion: 'Provide the user message to send to the agent'
+                });
+            }
+        }
+        catch (error) {
+            errors.push({
+                category: 'configuration',
+                message: `Configuration validation failed: ${error.message}`,
+                field: 'Configuration',
+                suggestion: 'Fix configuration JSON structure'
+            });
+        }
+        return {
+            valid: errors.length === 0,
+            errors,
+            warnings
+        };
+    }
+    /**
+     * Load agent entity.
+     * @private
+     */
+    async loadAgent(agentId, contextUser) {
+        const agent = await this._metadata.GetEntityObject('AI Agents', contextUser);
+        await agent.Load(agentId);
+        return agent;
+    }
+    /**
+     * Execute agent and return result.
+     * @private
+     */
+    async executeAgent(agent, input, contextUser, test, maxExecutionTime, testRun) {
+        const runner = new ai_agents_1.AgentRunner();
+        // Build conversation messages
+        const conversationMessages = [];
+        // Add prior messages if provided
+        if (input.conversationContext?.priorMessages) {
+            for (const msg of input.conversationContext.priorMessages) {
+                conversationMessages.push({
+                    role: msg.role,
+                    content: msg.content
+                });
+            }
+        }
+        // Add current user message
+        conversationMessages.push({
+            role: 'user',
+            content: input.userMessage
+        });
+        // Build execution parameters
+        const params = {
+            agent: agent, // Will be AIAgentEntityExtended at runtime
+            conversationMessages,
+            contextUser,
+            override: input.executionParams?.modelOverride ? {
+                modelId: input.executionParams.modelOverride
+            } : undefined
+        };
+        // Generate conversation name with sequence number (if in suite) or [Test] prefix (standalone)
+        // Sequence will be non-null when test is part of a suite run, showing test execution order
+        const conversationName = testRun.Sequence != null
+            ? `[${testRun.Sequence}] ${test.Name}`
+            : `[Test] ${test.Name}`;
+        // Execute agent with timeout if specified
+        if (maxExecutionTime) {
+            const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('Agent execution timeout')), maxExecutionTime));
+            const runResult = await Promise.race([
+                runner.RunAgentInConversation(params, {
+                    userMessage: input.userMessage,
+                    createArtifacts: true,
+                    conversationName: conversationName,
+                    testRunId: testRun.ID
+                }),
+                timeoutPromise
+            ]);
+            return { agentRun: runResult.agentResult.agentRun };
+        }
+        else {
+            const runResult = await runner.RunAgentInConversation(params, {
+                userMessage: input.userMessage,
+                createArtifacts: true,
+                conversationName: conversationName,
+                testRunId: testRun.ID
+            });
+            return { agentRun: runResult.agentResult.agentRun };
+        }
+    }
+    /**
+     * Create bidirectional link between TestRun and AgentRun.
+     * @private
+     */
+    async linkTestRunToAgentRun(testRun, agentRun) {
+        // Update AgentRun with hard FK to TestRun
+        agentRun.TestRunID = testRun.ID;
+        const saved = await agentRun.Save();
+        if (!saved) {
+            this.logError('Failed to link AgentRun to TestRun', new Error(agentRun.LatestResult?.Message));
+        }
+    }
+    /**
+     * Extract agent output from agent run.
+     * @private
+     */
+    extractAgentOutput(agentRun) {
+        return {
+            status: agentRun.Status,
+            success: agentRun.Success,
+            errorMessage: agentRun.ErrorMessage,
+            conversationId: agentRun.ConversationID
+        };
+    }
+    /**
+     * Run configured oracles.
+     * @private
+     */
+    async runOracles(config, input, expected, actualOutput, agentRun, context) {
+        const oracleResults = [];
+        // TODO: Temporarily skip oracle execution while oracles are being finalized
+        // Remove this flag once oracles are ready (SQL schema fixes, LLM Judge prompt creation, etc.)
+        const skipOracles = true;
+        if (skipOracles) {
+            this.log('⚠️  Oracle execution temporarily disabled', context.options.verbose);
+            return oracleResults;
+        }
+        for (const oracleConfig of config.oracles) {
+            const oracle = context.oracleRegistry.get(oracleConfig.type);
+            if (!oracle) {
+                this.logError(`Oracle not found: ${oracleConfig.type}`);
+                continue;
+            }
+            try {
+                const oracleInput = {
+                    test: context.test,
+                    expectedOutput: expected,
+                    actualOutput,
+                    targetEntity: agentRun,
+                    contextUser: context.contextUser
+                };
+                const result = await oracle.evaluate(oracleInput, oracleConfig.config || {});
+                oracleResults.push(result);
+                this.log(`Oracle ${oracleConfig.type}: ${result.passed ? 'PASSED' : 'FAILED'} (Score: ${result.score})`, context.options.verbose);
+            }
+            catch (error) {
+                this.logError(`Oracle ${oracleConfig.type} failed`, error);
+                oracleResults.push({
+                    oracleType: oracleConfig.type,
+                    passed: false,
+                    score: 0,
+                    message: `Oracle execution failed: ${error.message}`
+                });
+            }
+        }
+        return oracleResults;
+    }
+    /**
+     * Calculate total cost from agent run.
+     * @private
+     */
+    calculateTotalCost(agentRun) {
+        return agentRun.TotalCost || 0;
+    }
+    /**
+     * Calculate duration in milliseconds from agent run.
+     * @private
+     */
+    calculateDurationMs(agentRun) {
+        if (!agentRun.StartedAt || !agentRun.CompletedAt) {
+            return 0;
+        }
+        const start = new Date(agentRun.StartedAt).getTime();
+        const end = new Date(agentRun.CompletedAt).getTime();
+        return end - start;
+    }
+};
+exports.AgentEvalDriver = AgentEvalDriver;
+exports.AgentEvalDriver = AgentEvalDriver = __decorate([
+    (0, global_1.RegisterClass)(BaseTestDriver_1.BaseTestDriver, 'AgentEvalDriver')
+], AgentEvalDriver);
+//# sourceMappingURL=AgentEvalDriver.js.map

package/dist/drivers/AgentEvalDriver.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AgentEvalDriver.js","sourceRoot":"","sources":["../../src/drivers/AgentEvalDriver.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;AAGH,mDAAuD;AAEvD,yDAAwD;AAExD,qDAAkD;AA4GlD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAEI,IAAM,eAAe,GAArB,MAAM,eAAgB,SAAQ,+BAAc;IAC/C;;;;;;;;;;;;;OAaG;IACI,KAAK,CAAC,OAAO,CAAC,OAA+B;QAChD,IAAI,CAAC,GAAG,CAAC,2BAA2B,EAAE,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAE/D,IAAI,CAAC;YACD,sBAAsB;YACtB,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAkB,OAAO,CAAC,IAAI,CAAC,CAAC;YAC/D,MAAM,KAAK,GAAG,IAAI,CAAC,oBAAoB,CAAiB,OAAO,CAAC,IAAI,CAAC,CAAC;YACtE,MAAM,QAAQ,GAAG,IAAI,CAAC,qBAAqB,CAA4B,OAAO,CAAC,IAAI,CAAC,CAAC;YAErF,aAAa;YACb,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;YAExE,gBAAgB;YAChB,IAAI,CAAC,GAAG,CAAC,oBAAoB,KAAK,CAAC,IAAI,EAAE,EAAE,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACpE,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,YAAY,CACvC,KAAK,EACL,KAAK,EACL,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,IAAI,EACZ,MAAM,CAAC,gBAAgB,EACvB,OAAO,CAAC,OAAO,CAClB,CAAC;YAEF,MAAM,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC;YAEtC,4BAA4B;YAC5B,MAAM,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5D,wBAAwB;YACxB,MAAM,YAAY,GAAG,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;YAEvD,cAAc;YACd,IAAI,CAAC,GAAG,CAAC,gCAAgC,EAAE,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACpE,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,UAAU,CACvC,MAAM,EACN,KAAK,EACL,QAAQ,EACR,YAAY,EACZ,QAAQ,EACR,OAAO,CACV,CAAC;YAEF,6BAA6B;YAC7B,qEAAqE;YACrE,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,CAAC,aAAa,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC;YACxE,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,WAAW;gBACxE,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC;YAE1C,eAAe;YACf,MAAM,YAAY,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;YAChE,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC;YAEzC,iBAAiB;YACjB,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;YAEpD,2BAA2B;YAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;YAEtD,eAAe;YACf,MAAM,MAAM,GAA0B;gBAClC,UAAU,EAAE,UAAU;gBACtB,WAAW,EAAE,QAAQ,CAAC,EAAE;gBACxB,MAAM;gBACN,KAAK;gBACL,aAAa;gBACb,YAAY;gBACZ,YAAY,EAAE,WAAW,GAAG,YAAY;gBACxC,WAAW;gBACX,SAAS,EAAE,KAAK;gBAChB,cAAc,EAAE,QAAQ;gBACxB,YAAY;gBACZ,SAAS;gBACT,UAAU;aACb,CAAC;YAEF,IAAI,CAAC,GAAG,CACJ,+BAA+B,MAAM,YAAY,KAAK,GAAG,EACzD,OAAO,CAAC,OAAO,CAAC,OAAO,CAC1B,CAAC;YACF,OAAO,MAAM,CAAC;QAElB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,QAAQ,CAAC,yBAAyB,EAAE,KAAc,CAAC,CAAC;YACzD,MAAM,KAAK,CAAC;QAChB,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACa,KAAK,CAAC,QAAQ,CAAC,IAAgB;QAC3C,sBAAsB;QACtB,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC9C,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;YACpB,OAAO,UAAU,CAAC;QACtB,CAAC;QAED,MAAM,MAAM,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QAE1C,IAAI,CAAC;YACD,mCAAmC;YACnC,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAkB,IAAI,CAAC,CAAC;YAEvD,oBAAoB;YACpB,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBAClB,MAAM,CAAC,IAAI,CAAC;oBACR,QAAQ,EAAE,eAAe;oBACzB,OAAO,EAAE,sCAAsC;oBAC/C,KAAK,EAAE,uBAAuB;oBAC9B,UAAU,EAAE,qCAAqC;iBACpD,CAAC,CAAC;YACP,CAAC;YACD,+DAA+D;YAC/D,gDAAgD;YAEhD,wCAAwC;YACxC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjD,MAAM,CAAC,IAAI,CAAC;oBACR,QAAQ,EAAE,eAAe;oBACzB,OAAO,EAAE,iCAAiC;oBAC1C,KAAK,EAAE,uBAAuB;oBAC9B,UAAU,EAAE,8DAA8D;iBAC7E,CAAC,CAAC;YACP,CAAC;YACD,wEAAwE;YACxE,gDAAgD;YAEhD,2BAA2B;YAC3B,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;gBACxB,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,MAAM,CAC3D,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EACnB,CAAC,CACJ,CAAC;gBACF,IAAI,IAAI,CAAC,GAAG,CAAC,WAAW,GAAG,GAAG,CAAC,GAAG,IAAI,EAAE,CAAC;oBACrC,QAAQ,CAAC,IAAI,CAAC;wBACV,QAAQ,EAAE,eAAe;wBACzB,OAAO,EAAE,mCAAmC;wBAC5C,cAAc,EAAE,gBAAgB,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;qBAC3D,CAAC,CAAC;gBACP,CAAC;YACL,CAAC;YAED,4BAA4B;YAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,oBAAoB,CAAiB,IAAI,CAAC,CAAC;YAC9D,IAAI,CAAC,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;gBACxD,MAAM,CAAC,IAAI,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,4CAA4C;oBACrD,KAAK,EAAE,6BAA6B;oBACpC,UAAU,EAAE,+CAA+C;iBAC9D,CAAC,CAAC;YACP,CAAC;QAEL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC;gBACR,QAAQ,EAAE,eAAe;gBACzB,OAAO,EAAE,oCAAqC,KAAe,CAAC,OAAO,EAAE;gBACvE,KAAK,EAAE,eAAe;gBACtB,UAAU,EAAE,kCAAkC;aACjD,CAAC,CAAC;QACP,CAAC;QAED,OAAO;YACH,KAAK,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;YAC1B,MAAM;YACN,QAAQ;SACX,CAAC;IACN,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,SAAS,CAAC,OAAe,EAAE,WAAqB;QAC1D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,eAAe,CAAgB,WAAW,EAAE,WAAW,CAAC,CAAC;QAC5F,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC1B,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,YAAY,CACtB,KAAoB,EACpB,KAAqB,EACrB,WAAqB,EACrB,IAAgB,EAChB,gBAAoC,EACpC,OAAsB;QAEtB,MAAM,MAAM,GAAG,IAAI,uBAAW,EAAE,CAAC;QAEjC,8BAA8B;QAC9B,MAAM,oBAAoB,GAAkB,EAAE,CAAC;QAE/C,iCAAiC;QACjC,IAAI,KAAK,CAAC,mBAAmB,EAAE,aAAa,EAAE,CAAC;YAC3C,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,mBAAmB,CAAC,aAAa,EAAE,CAAC;gBACxD,oBAAoB,CAAC,IAAI,CAAC;oBACtB,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,OAAO,EAAE,GAAG,CAAC,OAAO;iBACR,CAAC,CAAC;YACtB,CAAC;QACL,CAAC;QAED,2BAA2B;QAC3B,oBAAoB,CAAC,IAAI,CAAC;YACtB,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,KAAK,CAAC,WAAW;SACd,CAAC,CAAC;QAElB,6BAA6B;QAC7B,MAAM,MAAM,GAAG;YACX,KAAK,EAAE,KAAY,EAAE,2CAA2C;YAChE,oBAAoB;YACpB,WAAW;YACX,QAAQ,EAAE,KAAK,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC,CAAC;gBAC7C,OAAO,EAAE,KAAK,CAAC,eAAe,CAAC,aAAa;aAC/C,CAAC,CAAC,CAAC,SAAS;SAChB,CAAC;QAEF,8FAA8F;QAC9F,2FAA2F;QAC3F,MAAM,gBAAgB,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI;YAC7C,CAAC,CAAC,IAAI,OAAO,CAAC,QAAQ,KAAK,IAAI,CAAC,IAAI,EAAE;YACtC,CAAC,CAAC,UAAU,IAAI,CAAC,IAAI,EAAE,CAAC;QAE5B,0CAA0C;QAC1C,IAAI,gBAAgB,EAAE,CAAC;YACnB,MAAM,cAAc,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CACpD,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC,EAAE,gBAAgB,CAAC,CACnF,CAAC;YAEF,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;gBACjC,MAAM,CAAC,sBAAsB,CAAC,MAAM,EAAE;oBAClC,WAAW,EAAE,KAAK,CAAC,WAAW;oBAC9B,eAAe,EAAE,IAAI;oBACrB,gBAAgB,EAAE,gBAAgB;oBAClC,SAAS,EAAE,OAAO,CAAC,EAAE;iBACxB,CAAC;gBACF,cAAc;aACjB,CAAC,CAAC;YAEH,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;QACxD,CAAC;aAAM,CAAC;YACJ,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,MAAM,EAAE;gBAC1D,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,eAAe,EAAE,IAAI;gBACrB,gBAAgB,EAAE,gBAAgB;gBAClC,SAAS,EAAE,OAAO,CAAC,EAAE;aACxB,CAAC,CAAC;YAEH,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;QACxD,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,qBAAqB,CAC/B,OAAsB,EACtB,QAA0B;QAE1B,0CAA0C;QAC1C,QAAQ,CAAC,SAAS,GAAG,OAAO,CAAC,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAEpC,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,IAAI,CAAC,QAAQ,CAAC,oCAAoC,EAAE,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;QACnG,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,kBAAkB,CAAC,QAA0B;QACjD,OAAO;YACH,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,YAAY,EAAE,QAAQ,CAAC,YAAY;YACnC,cAAc,EAAE,QAAQ,CAAC,cAAc;SAC1C,CAAC;IACN,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,UAAU,CACpB,MAAuB,EACvB,KAAqB,EACrB,QAAmC,EACnC,YAAqC,EACrC,QAA0B,EAC1B,OAA+B;QAE/B,MAAM,aAAa,GAAmB,EAAE,CAAC;QAEzC,4EAA4E;QAC5E,8FAA8F;QAC9F,MAAM,WAAW,GAAG,IAAI,CAAC;QAEzB,IAAI,WAAW,EAAE,CAAC;YACd,IAAI,CAAC,GAAG,CAAC,2CAA2C,EAAE,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC/E,OAAO,aAAa,CAAC;QACzB,CAAC;QAED,KAAK,MAAM,YAAY,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACxC,MAAM,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC7D,IAAI,CAAC,MAAM,EAAE,CAAC;gBACV,IAAI,CAAC,QAAQ,CAAC,qBAAqB,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;gBACxD,SAAS;YACb,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,WAAW,GAAgB;oBAC7B,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,cAAc,EAAE,QAAQ;oBACxB,YAAY;oBACZ,YAAY,EAAE,QAAQ;oBACtB,WAAW,EAAE,OAAO,CAAC,WAAW;iBACnC,CAAC;gBAEF,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,YAAY,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;gBAC7E,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAE3B,IAAI,CAAC,GAAG,CACJ,UAAU,YAAY,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,YAAY,MAAM,CAAC,KAAK,GAAG,EAC9F,OAAO,CAAC,OAAO,CAAC,OAAO,CAC1B,CAAC;YAEN,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,IAAI,CAAC,QAAQ,CAAC,UAAU,YAAY,CAAC,IAAI,SAAS,EAAE,KAAc,CAAC,CAAC;gBACpE,aAAa,CAAC,IAAI,CAAC;oBACf,UAAU,EAAE,YAAY,CAAC,IAAI;oBAC7B,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,4BAA6B,KAAe,CAAC,OAAO,EAAE;iBAClE,CAAC,CAAC;YACP,CAAC;QACL,CAAC;QAED,OAAO,aAAa,CAAC;IACzB,CAAC;IAED;;;OAGG;IACK,kBAAkB,CAAC,QAA0B;QACjD,OAAO,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;IACnC,CAAC;IAED;;;OAGG;IACK,mBAAmB,CAAC,QAA0B;QAClD,IAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YAC/C,OAAO,CAAC,CAAC;QACb,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;QACrD,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE,CAAC;QACrD,OAAO,GAAG,GAAG,KAAK,CAAC;IACvB,CAAC;CACJ,CAAA;AA3YY,0CAAe;0BAAf,eAAe;IAD3B,IAAA,sBAAa,EAAC,+BAAc,EAAE,iBAAiB,CAAC;GACpC,eAAe,CA2Y3B"}

package/dist/drivers/BaseTestDriver.d.ts

@@ -0,0 +1,145 @@
+/**
+ * @fileoverview Base class for all test driver implementations
+ * @module @memberjunction/testing-engine
+ */
+import { Metadata } from '@memberjunction/core';
+import { TestEntity } from '@memberjunction/core-entities';
+import { DriverExecutionContext, DriverExecutionResult, OracleResult, ScoringWeights, ValidationResult } from '../types';
+/**
+ * Abstract base class for test driver implementations.
+ *
+ * Each TestType in the database has a corresponding DriverClass that extends this base.
+ * The driver is responsible for:
+ * - Parsing test-specific configuration from Configuration JSON
+ * - Executing the test with appropriate logic
+ * - Running oracles to evaluate results
+ * - Calculating scores and determining pass/fail status
+ * - Returning structured results
+ *
+ * BaseTestDriver handles common functionality:
+ * - Configuration parsing
+ * - Score calculation
+ * - Status determination
+ * - Logging
+ * - Error handling
+ *
+ * Follows pattern from BaseScheduledJob and BaseAgent.
+ *
+ * @abstract
+ * @example
+ * ```typescript
+ * @RegisterClass(BaseTestDriver, 'AgentEvalDriver')
+ * export class AgentEvalDriver extends BaseTestDriver {
+ *     async Execute(context: DriverExecutionContext): Promise<DriverExecutionResult> {
+ *         const config = this.parseConfig<AgentEvalConfig>(context.test);
+ *         // Execute test logic
+ *         return result;
+ *     }
+ * }
+ * ```
+ */
+export declare abstract class BaseTestDriver {
+    protected _metadata: Metadata;
+    /**
+     * Execute the test.
+     *
+     * This is the main entry point for test execution. The driver should:
+     * 1. Parse Configuration, InputDefinition, ExpectedOutcomes from test entity
+     * 2. Perform test-specific execution (e.g., run agent, execute workflow)
+     * 3. Run oracles to evaluate results
+     * 4. Calculate score and determine status
+     * 5. Return structured DriverExecutionResult
+     *
+     * The base engine will handle:
+     * - Creating/updating TestRun entity
+     * - Logging to database
+     * - Error handling
+     * - Timing and cost tracking
+     *
+     * @param context - Execution context including test, run, user, options
+     * @returns Promise resolving to execution result
+     */
+    abstract Execute(context: DriverExecutionContext): Promise<DriverExecutionResult>;
+    /**
+     * Validate test configuration.
+     *
+     * Called when creating or updating a test to ensure the configuration is valid
+     * for this test type. Override to add type-specific validation.
+     *
+     * @param test - The test being validated
+     * @returns Validation result with errors and warnings
+     */
+    Validate(test: TestEntity): Promise<ValidationResult>;
+    /**
+     * Calculate overall score from oracle results.
+     *
+     * If weights are provided, calculates weighted average.
+     * Otherwise, calculates simple average.
+     *
+     * @param oracleResults - Results from oracle evaluations
+     * @param weights - Optional scoring weights by oracle type
+     * @returns Overall score from 0.0 to 1.0
+     * @protected
+     */
+    protected calculateScore(oracleResults: OracleResult[], weights?: ScoringWeights): number;
+    /**
+     * Determine overall test status from oracle results.
+     *
+     * Test passes only if ALL oracles pass.
+     *
+     * @param oracleResults - Results from oracle evaluations
+     * @returns 'Passed' if all oracles passed, 'Failed' otherwise
+     * @protected
+     */
+    protected determineStatus(oracleResults: OracleResult[]): 'Passed' | 'Failed';
+    /**
+     * Parse and validate Configuration JSON.
+     *
+     * Helper method for drivers to parse their configuration with type safety.
+     * Throws if configuration is invalid.
+     *
+     * @template T - The configuration type
+     * @param test - The test containing the configuration
+     * @returns Parsed configuration
+     * @throws Error if configuration is missing or invalid
+     * @protected
+     */
+    protected parseConfig<T>(test: TestEntity): T;
+    /**
+     * Parse and validate InputDefinition JSON.
+     *
+     * @template T - The input definition type
+     * @param test - The test containing the input definition
+     * @returns Parsed input definition
+     * @throws Error if input definition is missing or invalid
+     * @protected
+     */
+    protected parseInputDefinition<T>(test: TestEntity): T;
+    /**
+     * Parse and validate ExpectedOutcomes JSON.
+     *
+     * @template T - The expected outcomes type
+     * @param test - The test containing the expected outcomes
+     * @returns Parsed expected outcomes
+     * @throws Error if expected outcomes is missing or invalid
+     * @protected
+     */
+    protected parseExpectedOutcomes<T>(test: TestEntity): T;
+    /**
+     * Log execution progress.
+     *
+     * @param message - Log message
+     * @param verboseOnly - Whether to only log in verbose mode (default: false)
+     * @protected
+     */
+    protected log(message: string, verboseOnly?: boolean): void;
+    /**
+     * Log errors.
+     *
+     * @param message - Error message
+     * @param error - Optional error object
+     * @protected
+     */
+    protected logError(message: string, error?: Error): void;
+}
+//# sourceMappingURL=BaseTestDriver.d.ts.map

package/dist/drivers/BaseTestDriver.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"BaseTestDriver.d.ts","sourceRoot":"","sources":["../../src/drivers/BaseTestDriver.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAEH,QAAQ,EAIX,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACH,UAAU,EAEb,MAAM,+BAA+B,CAAC;AACvC,OAAO,EACH,sBAAsB,EACtB,qBAAqB,EACrB,YAAY,EACZ,cAAc,EACd,gBAAgB,EAGnB,MAAM,UAAU,CAAC;AAElB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,8BAAsB,cAAc;IAChC,SAAS,CAAC,SAAS,EAAE,QAAQ,CAAkB;IAE/C;;;;;;;;;;;;;;;;;;OAkBG;IACH,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAEjF;;;;;;;;OAQG;IACU,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAyElE;;;;;;;;;;OAUG;IACH,SAAS,CAAC,cAAc,CACpB,aAAa,EAAE,YAAY,EAAE,EAC7B,OAAO,CAAC,EAAE,cAAc,GACzB,MAAM;IAwBT;;;;;;;;OAQG;IACH,SAAS,CAAC,eAAe,CAAC,aAAa,EAAE,YAAY,EAAE,GAAG,QAAQ,GAAG,QAAQ;IAQ7E;;;;;;;;;;;OAWG;IACH,SAAS,CAAC,WAAW,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,GAAG,CAAC;IAa7C;;;;;;;;OAQG;IACH,SAAS,CAAC,oBAAoB,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,GAAG,CAAC;IAatD;;;;;;;;OAQG;IACH,SAAS,CAAC,qBAAqB,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,GAAG,CAAC;IAavD;;;;;;OAMG;IACH,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,GAAE,OAAe,GAAG,IAAI;IAQlE;;;;;;OAMG;IACH,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,GAAG,IAAI;CAG3D"}