@memberjunction/testing-engine 0.0.1 → 2.118.0

package/README.md

@@ -1,45 +1,419 @@
-# @memberjunction/testing-engine
+# MemberJunction Testing Engine
 
-## ⚠️ IMPORTANT NOTICE ⚠️
+Core execution engine for the MemberJunction Testing Framework. Provides the foundational infrastructure for running tests, evaluating results, and managing test execution.
 
-**This package is created solely for the purpose of setting up OIDC (OpenID Connect) trusted publishing with npm.**
+## Architecture
 
-This is **NOT** a functional package and contains **NO** code or functionality beyond the OIDC setup configuration.
+The Testing Engine follows a driver-based architecture that allows for extensible test types:
 
-## Purpose
+```
+TestEngine (Orchestration)
+    ↓
+BaseTestDriver (Abstract base class)
+    ↓
+Concrete Drivers:
+    - AgentEvalDriver
+    - WorkflowScenarioDriver (future)
+    - CodeGenTestDriver (future)
+    - IntegrationTestDriver (future)
+```
 
-This package exists to:
-1. Configure OIDC trusted publishing for the package name `@memberjunction/testing-engine`
-2. Enable secure, token-less publishing from CI/CD workflows
-3. Establish provenance for packages published under this name
+## Core Components
 
-## What is OIDC Trusted Publishing?
+### 1. Test Engine (`TestEngine`)
+Main orchestration layer responsible for:
+- Loading test definitions from database
+- Instantiating appropriate test drivers via ClassFactory
+- Managing test execution lifecycle
+- Aggregating and storing results
+- Tracking costs and performance metrics
 
-OIDC trusted publishing allows package maintainers to publish packages directly from their CI/CD workflows without needing to manage npm access tokens. Instead, it uses OpenID Connect to establish trust between the CI/CD provider (like GitHub Actions) and npm.
+### 2. Base Test Driver (`BaseTestDriver`)
+Abstract base class that all test drivers extend:
+- Defines execution contract
+- Provides common utilities (scoring, validation, result formatting)
+- Handles error management
+- Manages test context
 
-## Setup Instructions
+### 3. Oracle System
+Pluggable evaluation components:
+- **SchemaValidator** - Validates output structure
+- **TraceValidator** - Checks execution traces for errors
+- **LLMJudge** - Uses LLM to evaluate quality
+- **ExactMatcher** - Deterministic output comparison
+- **SQLValidator** - Database state verification
 
-To properly configure OIDC trusted publishing for this package:
+### 4. Result Management
+Structured result capture and storage:
+- TestRun entity management
+- Metric collection and aggregation
+- Artifact storage coordination
+- Cost tracking
 
-1. Go to [npmjs.com](https://www.npmjs.com/) and navigate to your package settings
-2. Configure the trusted publisher (e.g., GitHub Actions)
-3. Specify the repository and workflow that should be allowed to publish
-4. Use the configured workflow to publish your actual package
+## Class Structure
 
-## DO NOT USE THIS PACKAGE
+### TestEngine
 
-This package is a placeholder for OIDC configuration only. It:
-- Contains no executable code
-- Provides no functionality
-- Should not be installed as a dependency
-- Exists only for administrative purposes
+```typescript
+export class TestEngine {
+  /**
+   * Run a single test by ID
+   */
+  async runTest(
+    testId: string,
+    contextUser: UserInfo,
+    options?: TestRunOptions
+  ): Promise<TestRunResult>
 
-## More Information
+  /**
+   * Run an entire test suite
+   */
+  async runSuite(
+    suiteId: string,
+    contextUser: UserInfo,
+    options?: SuiteRunOptions
+  ): Promise<SuiteRunResult>
 
-For more details about npm's trusted publishing feature, see:
-- [npm Trusted Publishing Documentation](https://docs.npmjs.com/generating-provenance-statements)
-- [GitHub Actions OIDC Documentation](https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect)
+  /**
+   * Validate test definition without executing
+   */
+  async validateTest(testId: string): Promise<ValidationResult>
+}
+```
 
----
+### BaseTestDriver
 
-**Maintained for OIDC setup purposes only**
+```typescript
+export abstract class BaseTestDriver {
+  /**
+   * Execute the test and return results
+   * Must be implemented by concrete drivers
+   */
+  abstract execute(
+    test: TestEntity,
+    contextUser: UserInfo,
+    options: ExecutionOptions
+  ): Promise<DriverExecutionResult>
+
+  /**
+   * Validate test configuration
+   * Can be overridden by concrete drivers
+   */
+  async validate(test: TestEntity): Promise<ValidationResult>
+
+  /**
+   * Calculate overall score from oracle results
+   */
+  protected calculateScore(
+    oracleResults: OracleResult[],
+    weights: ScoringWeights
+  ): number
+
+  /**
+   * Store execution results to database
+   */
+  protected async storeResults(
+    testRun: TestRunEntity,
+    results: DriverExecutionResult
+  ): Promise<void>
+}
+```
+
+### Oracle Interface
+
+```typescript
+export interface IOracle {
+  /**
+   * Unique identifier for this oracle type
+   */
+  readonly type: string;
+
+  /**
+   * Execute the oracle evaluation
+   */
+  evaluate(
+    input: OracleInput,
+    config: OracleConfig
+  ): Promise<OracleResult>
+}
+
+export interface OracleResult {
+  oracleType: string;
+  passed: boolean;
+  score?: number;
+  message: string;
+  details?: any;
+}
+```
+
+## Agent Eval Driver
+
+The first concrete implementation:
+
+```typescript
+export class AgentEvalDriver extends BaseTestDriver {
+  async execute(
+    test: TestEntity,
+    contextUser: UserInfo,
+    options: ExecutionOptions
+  ): Promise<DriverExecutionResult> {
+    // 1. Parse test configuration
+    const config = this.parseConfig(test);
+
+    // 2. Execute agent with test inputs
+    const agentRun = await this.executeAgent(config.inputs, contextUser);
+
+    // 3. Run oracles to evaluate output
+    const oracleResults = await this.runOracles(
+      agentRun,
+      config.oracles,
+      config.expectedOutcomes
+    );
+
+    // 4. Calculate score based on weights
+    const score = this.calculateScore(oracleResults, config.weights);
+
+    // 5. Return structured results
+    return {
+      targetType: 'Agent Run',
+      targetLogId: agentRun.ID,
+      status: this.determineStatus(oracleResults),
+      score,
+      oracleResults,
+      cost: this.calculateCost(agentRun),
+      duration: this.calculateDuration(agentRun)
+    };
+  }
+
+  private async runOracles(
+    agentRun: AIAgentRunEntity,
+    oracleConfigs: OracleConfig[],
+    expectedOutcomes: any
+  ): Promise<OracleResult[]> {
+    const results: OracleResult[] = [];
+
+    for (const config of oracleConfigs) {
+      const oracle = this.getOracle(config.type);
+      const result = await oracle.evaluate({
+        agentRun,
+        expectedOutcomes
+      }, config);
+      results.push(result);
+    }
+
+    return results;
+  }
+}
+```
+
+## Oracle Implementations
+
+### Schema Validator
+```typescript
+export class SchemaValidatorOracle implements IOracle {
+  readonly type = 'schema-validate';
+
+  async evaluate(input: OracleInput, config: OracleConfig): Promise<OracleResult> {
+    const { agentRun } = input;
+    const schema = this.loadSchema(config.schema);
+
+    try {
+      schema.parse(agentRun.ResultPayload);
+      return {
+        oracleType: this.type,
+        passed: true,
+        score: 1.0,
+        message: 'Output matches schema'
+      };
+    } catch (error) {
+      return {
+        oracleType: this.type,
+        passed: false,
+        score: 0.0,
+        message: `Schema validation failed: ${error.message}`,
+        details: error.errors
+      };
+    }
+  }
+}
+```
+
+### Trace Validator
+```typescript
+export class TraceValidatorOracle implements IOracle {
+  readonly type = 'trace-no-errors';
+
+  async evaluate(input: OracleInput, config: OracleConfig): Promise<OracleResult> {
+    const { agentRun } = input;
+
+    // Load agent run steps
+    const rv = new RunView();
+    const stepsResult = await rv.RunView<AIAgentRunStepEntity>({
+      EntityName: 'MJ: AI Agent Run Steps',
+      ExtraFilter: `AgentRunID='${agentRun.ID}'`,
+      ResultType: 'entity_object'
+    });
+
+    if (!stepsResult.Success) {
+      throw new Error(`Failed to load agent run steps: ${stepsResult.ErrorMessage}`);
+    }
+
+    const steps = stepsResult.Results || [];
+    const errorSteps = steps.filter(s => s.Status === 'Error' || s.Status === 'Failed');
+
+    if (errorSteps.length === 0) {
+      return {
+        oracleType: this.type,
+        passed: true,
+        score: 1.0,
+        message: `All ${steps.length} steps completed without errors`
+      };
+    } else {
+      return {
+        oracleType: this.type,
+        passed: false,
+        score: 0.0,
+        message: `${errorSteps.length} of ${steps.length} steps had errors`,
+        details: errorSteps.map(s => ({
+          stepId: s.ID,
+          sequence: s.Sequence,
+          error: s.Notes
+        }))
+      };
+    }
+  }
+}
+```
+
+### LLM Judge
+```typescript
+export class LLMJudgeOracle implements IOracle {
+  readonly type = 'llm-judge';
+
+  async evaluate(input: OracleInput, config: OracleConfig): Promise<OracleResult> {
+    const { agentRun, expectedOutcomes } = input;
+
+    // Load rubric
+    const md = new Metadata();
+    const rubric = await md.GetEntityObject<TestRubricEntity>('Test Rubrics');
+    await rubric.Load(config.rubricId);
+
+    // Build evaluation prompt
+    const prompt = this.buildPrompt(rubric, agentRun, expectedOutcomes);
+
+    // Execute LLM evaluation
+    const judgmentResult = await this.executeJudgment(prompt);
+
+    // Parse structured response
+    const scores = this.parseScores(judgmentResult, rubric.Criteria);
+
+    // Calculate weighted score
+    const overallScore = this.calculateWeightedScore(scores, rubric.Criteria);
+
+    return {
+      oracleType: this.type,
+      passed: overallScore >= (config.threshold || 0.7),
+      score: overallScore,
+      message: `LLM judge score: ${(overallScore * 100).toFixed(1)}%`,
+      details: {
+        rubricId: rubric.ID,
+        rubricVersion: rubric.Version,
+        dimensionScores: scores,
+        judgmentNotes: judgmentResult.notes
+      }
+    };
+  }
+}
+```
+
+## Usage Example
+
+```typescript
+import { TestEngine } from '@memberjunction/testing-engine';
+import { getSystemUser } from '@memberjunction/core';
+
+// Initialize engine
+const engine = new TestEngine();
+
+// Run a single test
+const contextUser = getSystemUser();
+const result = await engine.runTest('test-id-123', contextUser, {
+  dryRun: false,
+  environment: 'staging'
+});
+
+console.log(`Test ${result.status}: Score ${result.score}`);
+console.log(`Cost: $${result.cost}, Duration: ${result.duration}s`);
+
+// Run a suite
+const suiteResult = await engine.runSuite('suite-id-456', contextUser, {
+  parallel: false,
+  failFast: true
+});
+
+console.log(`Suite completed: ${suiteResult.passedTests}/${suiteResult.totalTests} passed`);
+```
+
+## Extension Points
+
+### Adding New Test Types
+
+1. Create driver class extending `BaseTestDriver`:
+```typescript
+export class WorkflowScenarioDriver extends BaseTestDriver {
+  async execute(test: TestEntity, contextUser: UserInfo, options: ExecutionOptions) {
+    // Your workflow test logic
+  }
+}
+```
+
+2. Register with ClassFactory:
+```typescript
+MJGlobal.Instance.ClassFactory.Register(
+  BaseTestDriver,
+  'WorkflowScenarioDriver',
+  WorkflowScenarioDriver
+);
+```
+
+3. Create TestType record in database:
+```sql
+INSERT INTO TestType (Name, DriverClass, Status)
+VALUES ('Workflow Scenario', 'WorkflowScenarioDriver', 'Active');
+```
+
+### Adding New Oracles
+
+1. Implement `IOracle` interface:
+```typescript
+export class CustomOracle implements IOracle {
+  readonly type = 'custom-check';
+
+  async evaluate(input: OracleInput, config: OracleConfig): Promise<OracleResult> {
+    // Your evaluation logic
+  }
+}
+```
+
+2. Register with engine:
+```typescript
+engine.registerOracle(new CustomOracle());
+```
+
+## Testing Best Practices
+
+1. **Use appropriate oracles** - Combine deterministic checks (schema, trace) with semantic checks (LLM judge)
+2. **Set realistic weights** - Balance different evaluation dimensions
+3. **Track costs** - Monitor LLM evaluation costs
+4. **Validate before execute** - Use `validateTest()` to catch config errors
+5. **Leverage traces** - Link TestRun to target entities for debugging
+
+## Future Enhancements
+
+- [ ] Parallel test execution within suites
+- [ ] Test retry logic with exponential backoff
+- [ ] Result caching for expensive evaluations
+- [ ] Composite oracles (AND/OR logic)
+- [ ] Dataset-driven test execution (run same test with multiple inputs)
+- [ ] Comparative evaluation (A/B testing different agent versions)
+- [ ] Replay mode (re-evaluate existing agent runs)

package/dist/drivers/AgentEvalDriver.d.ts

@@ -0,0 +1,197 @@
+/**
+ * @fileoverview Test driver for AI Agent evaluation
+ * @module @memberjunction/testing-engine
+ */
+import { TestEntity } from '@memberjunction/core-entities';
+import { BaseTestDriver } from './BaseTestDriver';
+import { DriverExecutionContext, DriverExecutionResult, ValidationResult } from '../types';
+/**
+ * Configuration for Agent Evaluation tests.
+ */
+export interface AgentEvalConfig {
+    /**
+     * Agent to test
+     */
+    agentId: string;
+    /**
+     * Oracles to run for evaluation
+     */
+    oracles: {
+        type: string;
+        config?: Record<string, unknown>;
+        weight?: number;
+    }[];
+    /**
+     * Scoring weights by oracle type
+     */
+    scoringWeights?: Record<string, number>;
+    /**
+     * Maximum execution time in milliseconds
+     */
+    maxExecutionTime?: number;
+}
+/**
+ * Input definition for Agent Evaluation tests.
+ */
+export interface AgentEvalInput {
+    /**
+     * User message to send to agent
+     */
+    userMessage: string;
+    /**
+     * Optional conversation context
+     */
+    conversationContext?: {
+        conversationId?: string;
+        priorMessages?: Array<{
+            role: 'user' | 'assistant';
+            content: string;
+        }>;
+    };
+    /**
+     * Optional agent execution parameters
+     */
+    executionParams?: {
+        modelOverride?: string;
+        temperatureOverride?: number;
+        maxTokensOverride?: number;
+    };
+}
+/**
+ * Expected outcomes for Agent Evaluation tests.
+ */
+export interface AgentEvalExpectedOutcomes {
+    /**
+     * Expected response patterns (for regex matching)
+     */
+    responsePatterns?: string[];
+    /**
+     * Expected entities mentioned in response
+     */
+    expectedEntities?: string[];
+    /**
+     * Expected actions taken
+     */
+    expectedActions?: string[];
+    /**
+     * Schema for response validation
+     */
+    responseSchema?: Record<string, unknown>;
+    /**
+     * SQL queries to validate database state
+     */
+    sqlValidations?: Array<{
+        query: string;
+        expectedResult: unknown;
+    }>;
+    /**
+     * Custom validation criteria for LLM judge
+     */
+    judgeValidationCriteria?: string[];
+}
+/**
+ * Test driver for AI Agent evaluation.
+ *
+ * Executes an AI agent with test input, runs configured oracles,
+ * and creates bidirectional link between TestRun and AgentRun.
+ *
+ * @example
+ * ```typescript
+ * // Configuration JSON in Test entity
+ * {
+ *   "agentId": "agent-123",
+ *   "oracles": [
+ *     { "type": "trace-no-errors", "weight": 0.2 },
+ *     { "type": "llm-judge", "weight": 0.5, "config": { "criteria": [...] } },
+ *     { "type": "schema-validate", "weight": 0.3, "config": { "schema": {...} } }
+ *   ],
+ *   "scoringWeights": { "trace-no-errors": 0.2, "llm-judge": 0.5, "schema-validate": 0.3 }
+ * }
+ *
+ * // InputDefinition JSON in Test entity
+ * {
+ *   "userMessage": "Create a report showing sales by region",
+ *   "conversationContext": null,
+ *   "executionParams": { "temperatureOverride": 0.3 }
+ * }
+ *
+ * // ExpectedOutcomes JSON in Test entity
+ * {
+ *   "responsePatterns": ["sales.*region", "chart|graph"],
+ *   "expectedEntities": ["Report", "Dashboard"],
+ *   "responseSchema": { "type": "object", "properties": {...} },
+ *   "judgeValidationCriteria": [
+ *     "Response accurately answers the user's question",
+ *     "Report includes proper data visualization",
+ *     "Response is professional and clear"
+ *   ]
+ * }
+ * ```
+ */
+export declare class AgentEvalDriver extends BaseTestDriver {
+    /**
+     * Execute agent evaluation test.
+     *
+     * Steps:
+     * 1. Parse configuration and input
+     * 2. Load and execute agent via AgentRunner
+     * 3. Create bidirectional link (TestRun ↔ AgentRun)
+     * 4. Run oracles to evaluate results
+     * 5. Calculate score and determine status
+     * 6. Return structured results
+     *
+     * @param context - Execution context
+     * @returns Execution result
+     */
+    Execute(context: DriverExecutionContext): Promise<DriverExecutionResult>;
+    /**
+     * Validate agent evaluation test configuration.
+     *
+     * Checks:
+     * - Base validation (InputDefinition, ExpectedOutcomes, Configuration)
+     * - Agent ID is valid
+     * - At least one oracle is configured
+     * - Oracle types are registered
+     * - Scoring weights are valid
+     *
+     * @param test - Test entity to validate
+     * @returns Validation result
+     */
+    Validate(test: TestEntity): Promise<ValidationResult>;
+    /**
+     * Load agent entity.
+     * @private
+     */
+    private loadAgent;
+    /**
+     * Execute agent and return result.
+     * @private
+     */
+    private executeAgent;
+    /**
+     * Create bidirectional link between TestRun and AgentRun.
+     * @private
+     */
+    private linkTestRunToAgentRun;
+    /**
+     * Extract agent output from agent run.
+     * @private
+     */
+    private extractAgentOutput;
+    /**
+     * Run configured oracles.
+     * @private
+     */
+    private runOracles;
+    /**
+     * Calculate total cost from agent run.
+     * @private
+     */
+    private calculateTotalCost;
+    /**
+     * Calculate duration in milliseconds from agent run.
+     * @private
+     */
+    private calculateDurationMs;
+}
+//# sourceMappingURL=AgentEvalDriver.d.ts.map

package/dist/drivers/AgentEvalDriver.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AgentEvalDriver.d.ts","sourceRoot":"","sources":["../../src/drivers/AgentEvalDriver.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EAAmC,UAAU,EAAiB,MAAM,+BAA+B,CAAC;AAG3G,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EACH,sBAAsB,EACtB,qBAAqB,EAGrB,gBAAgB,EAGnB,MAAM,UAAU,CAAC;AAElB;;GAEG;AACH,MAAM,WAAW,eAAe;IAC5B;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,OAAO,EAAE;QACL,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,EAAE,CAAC;IAEJ;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAExC;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,mBAAmB,CAAC,EAAE;QAClB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,aAAa,CAAC,EAAE,KAAK,CAAC;YAClB,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;YAC3B,OAAO,EAAE,MAAM,CAAC;SACnB,CAAC,CAAC;KACN,CAAC;IAEF;;OAEG;IACH,eAAe,CAAC,EAAE;QACd,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC9B,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACtC;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE5B;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE5B;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAE3B;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEzC;;OAEG;IACH,cAAc,CAAC,EAAE,KAAK,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;QACd,cAAc,EAAE,OAAO,CAAC;KAC3B,CAAC,CAAC;IAEH;;OAEG;IACH,uBAAuB,CAAC,EAAE,MAAM,EAAE,CAAC;CACtC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,qBACa,eAAgB,SAAQ,cAAc;IAC/C;;;;;;;;;;;;;OAaG;IACU,OAAO,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAwFrF;;;;;;;;;;;;OAYG;IACmB,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAgF3E;;;OAGG;YACW,SAAS;IAMvB;;;OAGG;YACW,YAAY;IA0E1B;;;OAGG;YACW,qBAAqB;IAanC;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAS1B;;;OAGG;YACW,UAAU;IAyDxB;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAI1B;;;OAGG;IACH,OAAO,CAAC,mBAAmB;CAQ9B"}