@memberjunction/testing-engine 0.0.1 → 2.119.0

package/dist/types.d.ts

@@ -0,0 +1,428 @@
+/**
+ * Core type definitions for the Testing Engine
+ */
+import { UserInfo } from '@memberjunction/core';
+import { TestEntity, TestRunEntity } from '@memberjunction/core-entities';
+import { IOracle } from './oracles/IOracle';
+/**
+ * Progress callback for test execution
+ */
+export interface TestProgress {
+    /**
+     * Current execution step
+     */
+    step: string;
+    /**
+     * Progress percentage (0-100)
+     */
+    percentage: number;
+    /**
+     * Human-readable message
+     */
+    message: string;
+    /**
+     * Additional metadata
+     */
+    metadata?: {
+        testName?: string;
+        testRun?: any;
+        driverType?: string;
+        oracleType?: string;
+        [key: string]: any;
+    };
+}
+/**
+ * Options for running a single test
+ */
+export interface TestRunOptions {
+    /**
+     * Verbose logging
+     */
+    verbose?: boolean;
+    /**
+     * Validate configuration without executing
+     */
+    dryRun?: boolean;
+    /**
+     * Environment context (dev, staging, prod)
+     */
+    environment?: string;
+    /**
+     * Git commit SHA for versioning
+     */
+    gitCommit?: string;
+    /**
+     * Agent/system version being tested
+     */
+    agentVersion?: string;
+    /**
+     * Override test configuration
+     */
+    configOverride?: Record<string, unknown>;
+    /**
+     * Progress callback for real-time updates
+     */
+    progressCallback?: (progress: TestProgress) => void;
+}
+/**
+ * Options for running a test suite
+ */
+export interface SuiteRunOptions extends TestRunOptions {
+    /**
+     * Run tests in parallel
+     */
+    parallel?: boolean;
+    /**
+     * Stop on first failure
+     */
+    failFast?: boolean;
+    /**
+     * Maximum parallel tests (if parallel=true)
+     */
+    maxParallel?: number;
+    /**
+     * Run only specific sequence numbers
+     */
+    sequence?: number[];
+}
+/**
+ * Result from running a single test
+ */
+export interface TestRunResult {
+    /**
+     * Test Run ID
+     */
+    testRunId: string;
+    /**
+     * Test ID
+     */
+    testId: string;
+    /**
+     * Test name (from lookup field)
+     */
+    testName: string;
+    /**
+     * Test execution status
+     */
+    status: 'Passed' | 'Failed' | 'Skipped' | 'Error';
+    /**
+     * Overall score (0.0000 to 1.0000)
+     */
+    score: number;
+    /**
+     * Number of checks that passed
+     */
+    passedChecks: number;
+    /**
+     * Number of checks that failed
+     */
+    failedChecks: number;
+    /**
+     * Total number of checks
+     */
+    totalChecks: number;
+    /**
+     * Oracle evaluation results
+     */
+    oracleResults: OracleResult[];
+    /**
+     * Target entity type (e.g., "AI Agent")
+     */
+    targetType: string;
+    /**
+     * Target entity ID (e.g., AIAgentRun.ID)
+     */
+    targetLogId: string;
+    /**
+     * Execution duration in milliseconds
+     */
+    durationMs: number;
+    /**
+     * Cost in USD
+     */
+    totalCost: number;
+    /**
+     * When execution started
+     */
+    startedAt: Date;
+    /**
+     * When execution completed
+     */
+    completedAt: Date;
+    /**
+     * Error message if status is Error
+     */
+    errorMessage?: string;
+    /**
+     * Iteration number for repeated tests (when RepeatCount > 1)
+     */
+    sequence?: number;
+}
+/**
+ * Result from running a test suite
+ */
+export interface TestSuiteRunResult {
+    /**
+     * Suite Run ID
+     */
+    suiteRunId: string;
+    /**
+     * Suite ID
+     */
+    suiteId: string;
+    /**
+     * Suite name (from lookup field)
+     */
+    suiteName: string;
+    /**
+     * Suite execution status
+     */
+    status: 'Completed' | 'Failed' | 'Cancelled' | 'Pending' | 'Running';
+    /**
+     * Tests that passed
+     */
+    passedTests: number;
+    /**
+     * Tests that failed
+     */
+    failedTests: number;
+    /**
+     * Total tests
+     */
+    totalTests: number;
+    /**
+     * Average score across all tests
+     */
+    averageScore: number;
+    /**
+     * Individual test results
+     */
+    testResults: TestRunResult[];
+    /**
+     * Total duration in milliseconds
+     */
+    durationMs: number;
+    /**
+     * Total cost in USD
+     */
+    totalCost: number;
+    /**
+     * When execution started
+     */
+    startedAt: Date;
+    /**
+     * When execution completed
+     */
+    completedAt: Date;
+}
+/**
+ * Context for test driver execution
+ */
+export interface DriverExecutionContext {
+    /**
+     * Test definition
+     */
+    test: TestEntity;
+    /**
+     * Test run entity (for bidirectional linking)
+     */
+    testRun: TestRunEntity;
+    /**
+     * User context for data access
+     */
+    contextUser: UserInfo;
+    /**
+     * Runtime options
+     */
+    options: TestRunOptions;
+    /**
+     * Oracle registry for evaluations
+     */
+    oracleRegistry: Map<string, IOracle>;
+}
+/**
+ * Result from driver execution
+ */
+export interface DriverExecutionResult {
+    /**
+     * Target entity type (e.g., "AI Agent")
+     */
+    targetType: string;
+    /**
+     * Target entity ID
+     */
+    targetLogId: string;
+    /**
+     * Execution status
+     */
+    status: 'Passed' | 'Failed' | 'Error';
+    /**
+     * Overall score
+     */
+    score: number;
+    /**
+     * Oracle results
+     */
+    oracleResults: OracleResult[];
+    /**
+     * Number of checks that passed
+     */
+    passedChecks: number;
+    /**
+     * Number of checks that failed
+     */
+    failedChecks: number;
+    /**
+     * Total number of checks
+     */
+    totalChecks: number;
+    /**
+     * Input data used
+     */
+    inputData?: unknown;
+    /**
+     * Expected output data
+     */
+    expectedOutput?: unknown;
+    /**
+     * Actual output data
+     */
+    actualOutput?: unknown;
+    /**
+     * Cost in USD
+     */
+    totalCost?: number;
+    /**
+     * Duration in milliseconds
+     */
+    durationMs?: number;
+    /**
+     * Error message if status is Error
+     */
+    errorMessage?: string;
+}
+/**
+ * Oracle evaluation input
+ */
+export interface OracleInput {
+    /**
+     * The test being evaluated
+     */
+    test: TestEntity;
+    /**
+     * Expected output from test definition
+     */
+    expectedOutput?: unknown;
+    /**
+     * Actual output from execution
+     */
+    actualOutput?: unknown;
+    /**
+     * Target entity (e.g., AgentRun)
+     */
+    targetEntity?: unknown;
+    /**
+     * User context
+     */
+    contextUser: UserInfo;
+}
+/**
+ * Oracle configuration (can have any additional properties)
+ */
+export interface OracleConfig {
+    /**
+     * Oracle-specific configuration properties
+     */
+    [key: string]: unknown;
+}
+/**
+ * Oracle evaluation result
+ */
+export interface OracleResult {
+    /**
+     * Oracle type that produced this result
+     */
+    oracleType: string;
+    /**
+     * Whether the oracle check passed
+     */
+    passed: boolean;
+    /**
+     * Numeric score (0.0 to 1.0)
+     */
+    score: number;
+    /**
+     * Human-readable message
+     */
+    message: string;
+    /**
+     * Additional details (oracle-specific)
+     */
+    details?: unknown;
+}
+/**
+ * Scoring weights for different evaluation dimensions
+ */
+export interface ScoringWeights {
+    /**
+     * Weight for each oracle type
+     * Keys are oracle types, values are weights (should sum to 1.0)
+     */
+    [oracleType: string]: number;
+}
+/**
+ * Validation result for test configuration
+ */
+export interface ValidationResult {
+    /**
+     * Whether validation passed
+     */
+    valid: boolean;
+    /**
+     * Validation errors (blocking issues)
+     */
+    errors: ValidationError[];
+    /**
+     * Validation warnings (non-blocking issues)
+     */
+    warnings: ValidationWarning[];
+}
+/**
+ * Validation error
+ */
+export interface ValidationError {
+    /**
+     * Error category
+     */
+    category: 'configuration' | 'input' | 'expected-outcome';
+    /**
+     * Error message
+     */
+    message: string;
+    /**
+     * Field path (if applicable)
+     */
+    field?: string;
+    /**
+     * Suggested fix
+     */
+    suggestion?: string;
+}
+/**
+ * Validation warning
+ */
+export interface ValidationWarning {
+    /**
+     * Warning category
+     */
+    category: 'best-practice' | 'performance' | 'cost';
+    /**
+     * Warning message
+     */
+    message: string;
+    /**
+     * Recommendation
+     */
+    recommendation?: string;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EACL,UAAU,EACV,aAAa,EAKd,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAE5C;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,QAAQ,CAAC,EAAE;QACT,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,OAAO,CAAC,EAAE,GAAG,CAAC;QACd,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;KACpB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IAEjB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEzC;;OAEG;IACH,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;CACrD;AAED;;GAEG;AACH,MAAM,WAAW,eAAgB,SAAQ,cAAc;IACrD;;OAEG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;OAEG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;IAElD;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,aAAa,EAAE,YAAY,EAAE,CAAC;IAE9B;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,SAAS,EAAE,IAAI,CAAC;IAEhB;;OAEG;IACH,WAAW,EAAE,IAAI,CAAC;IAElB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,MAAM,EAAE,WAAW,GAAG,QAAQ,GAAG,WAAW,GAAG,SAAS,GAAG,SAAS,CAAC;IAErE;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,WAAW,EAAE,aAAa,EAAE,CAAC;IAE7B;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,SAAS,EAAE,IAAI,CAAC;IAEhB;;OAEG;IACH,WAAW,EAAE,IAAI,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC;;OAEG;IACH,IAAI,EAAE,UAAU,CAAC;IAEjB;;OAEG;IACH,OAAO,EAAE,aAAa,CAAC;IAEvB;;OAEG;IACH,WAAW,EAAE,QAAQ,CAAC;IAEtB;;OAEG;IACH,OAAO,EAAE,cAAc,CAAC;IAExB;;OAEG;IACH,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;IAEtC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,aAAa,EAAE,YAAY,EAAE,CAAC;IAE9B;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;OAEG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B;;OAEG;IACH,IAAI,EAAE,UAAU,CAAC;IAEjB;;OAEG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB;;OAEG;IACH,WAAW,EAAE,QAAQ,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B;;OAEG;IACH,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,MAAM,EAAE,OAAO,CAAC;IAEhB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;OAEG;IACH,KAAK,EAAE,OAAO,CAAC;IAEf;;OAEG;IACH,MAAM,EAAE,eAAe,EAAE,CAAC;IAE1B;;OAEG;IACH,QAAQ,EAAE,iBAAiB,EAAE,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;OAEG;IACH,QAAQ,EAAE,eAAe,GAAG,OAAO,GAAG,kBAAkB,CAAC;IAEzD;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,QAAQ,EAAE,eAAe,GAAG,aAAa,GAAG,MAAM,CAAC;IAEnD;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB"}

package/dist/types.js

@@ -0,0 +1,6 @@
+"use strict";
+/**
+ * Core type definitions for the Testing Engine
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":";AAAA;;GAEG"}

package/dist/utils/cost-calculator.d.ts

@@ -0,0 +1,92 @@
+/**
+ * @fileoverview Cost calculation utilities for test execution
+ * @module @memberjunction/testing-engine
+ */
+import { UserInfo } from '@memberjunction/core';
+import { AIAgentRunEntity } from '@memberjunction/core-entities';
+/**
+ * Calculate total cost from AI Agent Run.
+ *
+ * Aggregates costs from all AI prompt runs within the agent run.
+ *
+ * @param agentRun - Agent run entity
+ * @param contextUser - User context
+ * @returns Total cost in dollars
+ */
+export declare function calculateAgentRunCost(agentRun: AIAgentRunEntity, contextUser: UserInfo): Promise<number>;
+/**
+ * Calculate cost from multiple test runs.
+ *
+ * @param testRunCosts - Array of individual test run costs
+ * @returns Total cost
+ */
+export declare function aggregateTestRunCosts(testRunCosts: number[]): number;
+/**
+ * Calculate average cost per test.
+ *
+ * @param totalCost - Total cost across all tests
+ * @param testCount - Number of tests
+ * @returns Average cost per test
+ */
+export declare function calculateAverageCost(totalCost: number, testCount: number): number;
+/**
+ * Format cost for display.
+ *
+ * @param cost - Cost in dollars
+ * @param precision - Decimal places (default: 4)
+ * @returns Formatted cost string
+ */
+export declare function formatCost(cost: number, precision?: number): string;
+/**
+ * Calculate cost per oracle evaluation.
+ *
+ * Used for tracking costs of LLM-based oracles.
+ *
+ * @param oracleResults - Oracle results with cost details
+ * @returns Map of oracle type to cost
+ */
+export declare function calculateOracleCosts(oracleResults: Array<{
+    oracleType: string;
+    details?: {
+        llmCost?: number;
+    };
+}>): Map<string, number>;
+/**
+ * Calculate cost breakdown by component.
+ *
+ * @param testRunCost - Total test run cost
+ * @param oracleCosts - Costs by oracle type
+ * @returns Cost breakdown
+ */
+export declare function calculateCostBreakdown(testRunCost: number, oracleCosts: Map<string, number>): {
+    agentExecution: number;
+    oracleEvaluation: number;
+    total: number;
+    breakdown: Array<{
+        component: string;
+        cost: number;
+        percentage: number;
+    }>;
+};
+/**
+ * Estimate cost for test suite.
+ *
+ * @param testCount - Number of tests
+ * @param avgCostPerTest - Average cost per test
+ * @returns Estimated total cost
+ */
+export declare function estimateSuiteCost(testCount: number, avgCostPerTest: number): number;
+/**
+ * Calculate cost efficiency metrics.
+ *
+ * @param totalCost - Total execution cost
+ * @param passedTests - Number of passed tests
+ * @param totalTests - Total number of tests
+ * @returns Cost efficiency metrics
+ */
+export declare function calculateCostEfficiency(totalCost: number, passedTests: number, totalTests: number): {
+    costPerTest: number;
+    costPerPassedTest: number;
+    costEfficiencyRatio: number;
+};
+//# sourceMappingURL=cost-calculator.d.ts.map

package/dist/utils/cost-calculator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cost-calculator.d.ts","sourceRoot":"","sources":["../../src/utils/cost-calculator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAW,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAqB,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AAEpF;;;;;;;;GAQG;AACH,wBAAsB,qBAAqB,CACvC,QAAQ,EAAE,gBAAgB,EAC1B,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,MAAM,CAAC,CAGjB;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,MAAM,CAEpE;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAEjF;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,GAAE,MAAU,GAAG,MAAM,CAUtE;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAChC,aAAa,EAAE,KAAK,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAClC,CAAC,GACH,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CASrB;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAClC,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GACjC;IACC,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC7E,CAuBA;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,MAAM,CAEnF;AAED;;;;;;;GAOG;AACH,wBAAgB,uBAAuB,CACnC,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,GACnB;IACC,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;CAC/B,CAUA"}

package/dist/utils/cost-calculator.js

@@ -0,0 +1,137 @@
+"use strict";
+/**
+ * @fileoverview Cost calculation utilities for test execution
+ * @module @memberjunction/testing-engine
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.calculateCostEfficiency = exports.estimateSuiteCost = exports.calculateCostBreakdown = exports.calculateOracleCosts = exports.formatCost = exports.calculateAverageCost = exports.aggregateTestRunCosts = exports.calculateAgentRunCost = void 0;
+/**
+ * Calculate total cost from AI Agent Run.
+ *
+ * Aggregates costs from all AI prompt runs within the agent run.
+ *
+ * @param agentRun - Agent run entity
+ * @param contextUser - User context
+ * @returns Total cost in dollars
+ */
+async function calculateAgentRunCost(agentRun, contextUser) {
+    // Agent run already has TotalCost calculated
+    return agentRun.TotalCost || 0;
+}
+exports.calculateAgentRunCost = calculateAgentRunCost;
+/**
+ * Calculate cost from multiple test runs.
+ *
+ * @param testRunCosts - Array of individual test run costs
+ * @returns Total cost
+ */
+function aggregateTestRunCosts(testRunCosts) {
+    return testRunCosts.reduce((sum, cost) => sum + cost, 0);
+}
+exports.aggregateTestRunCosts = aggregateTestRunCosts;
+/**
+ * Calculate average cost per test.
+ *
+ * @param totalCost - Total cost across all tests
+ * @param testCount - Number of tests
+ * @returns Average cost per test
+ */
+function calculateAverageCost(totalCost, testCount) {
+    return testCount > 0 ? totalCost / testCount : 0;
+}
+exports.calculateAverageCost = calculateAverageCost;
+/**
+ * Format cost for display.
+ *
+ * @param cost - Cost in dollars
+ * @param precision - Decimal places (default: 4)
+ * @returns Formatted cost string
+ */
+function formatCost(cost, precision = 4) {
+    if (cost === 0) {
+        return '$0.00';
+    }
+    if (cost < 0.0001) {
+        return `$${cost.toExponential(2)}`;
+    }
+    return `$${cost.toFixed(precision)}`;
+}
+exports.formatCost = formatCost;
+/**
+ * Calculate cost per oracle evaluation.
+ *
+ * Used for tracking costs of LLM-based oracles.
+ *
+ * @param oracleResults - Oracle results with cost details
+ * @returns Map of oracle type to cost
+ */
+function calculateOracleCosts(oracleResults) {
+    const costs = new Map();
+    for (const result of oracleResults) {
+        const cost = result.details?.llmCost || 0;
+        costs.set(result.oracleType, cost);
+    }
+    return costs;
+}
+exports.calculateOracleCosts = calculateOracleCosts;
+/**
+ * Calculate cost breakdown by component.
+ *
+ * @param testRunCost - Total test run cost
+ * @param oracleCosts - Costs by oracle type
+ * @returns Cost breakdown
+ */
+function calculateCostBreakdown(testRunCost, oracleCosts) {
+    const oracleTotal = Array.from(oracleCosts.values()).reduce((sum, c) => sum + c, 0);
+    const agentExecution = Math.max(0, testRunCost - oracleTotal);
+    const breakdown = [
+        {
+            component: 'Agent Execution',
+            cost: agentExecution,
+            percentage: testRunCost > 0 ? (agentExecution / testRunCost) * 100 : 0
+        },
+        ...Array.from(oracleCosts.entries()).map(([type, cost]) => ({
+            component: `Oracle: ${type}`,
+            cost,
+            percentage: testRunCost > 0 ? (cost / testRunCost) * 100 : 0
+        }))
+    ];
+    return {
+        agentExecution,
+        oracleEvaluation: oracleTotal,
+        total: testRunCost,
+        breakdown
+    };
+}
+exports.calculateCostBreakdown = calculateCostBreakdown;
+/**
+ * Estimate cost for test suite.
+ *
+ * @param testCount - Number of tests
+ * @param avgCostPerTest - Average cost per test
+ * @returns Estimated total cost
+ */
+function estimateSuiteCost(testCount, avgCostPerTest) {
+    return testCount * avgCostPerTest;
+}
+exports.estimateSuiteCost = estimateSuiteCost;
+/**
+ * Calculate cost efficiency metrics.
+ *
+ * @param totalCost - Total execution cost
+ * @param passedTests - Number of passed tests
+ * @param totalTests - Total number of tests
+ * @returns Cost efficiency metrics
+ */
+function calculateCostEfficiency(totalCost, passedTests, totalTests) {
+    const costPerTest = totalTests > 0 ? totalCost / totalTests : 0;
+    const costPerPassedTest = passedTests > 0 ? totalCost / passedTests : 0;
+    const costEfficiencyRatio = totalCost > 0 ? passedTests / totalCost : 0;
+    return {
+        costPerTest,
+        costPerPassedTest,
+        costEfficiencyRatio
+    };
+}
+exports.calculateCostEfficiency = calculateCostEfficiency;
+//# sourceMappingURL=cost-calculator.js.map

package/dist/utils/cost-calculator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cost-calculator.js","sourceRoot":"","sources":["../../src/utils/cost-calculator.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAKH;;;;;;;;GAQG;AACI,KAAK,UAAU,qBAAqB,CACvC,QAA0B,EAC1B,WAAqB;IAErB,6CAA6C;IAC7C,OAAO,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;AACnC,CAAC;AAND,sDAMC;AAED;;;;;GAKG;AACH,SAAgB,qBAAqB,CAAC,YAAsB;IACxD,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC;AAC7D,CAAC;AAFD,sDAEC;AAED;;;;;;GAMG;AACH,SAAgB,oBAAoB,CAAC,SAAiB,EAAE,SAAiB;IACrE,OAAO,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;AACrD,CAAC;AAFD,oDAEC;AAED;;;;;;GAMG;AACH,SAAgB,UAAU,CAAC,IAAY,EAAE,YAAoB,CAAC;IAC1D,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;QACb,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,IAAI,IAAI,GAAG,MAAM,EAAE,CAAC;QAChB,OAAO,IAAI,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;IACvC,CAAC;IAED,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;AACzC,CAAC;AAVD,gCAUC;AAED;;;;;;;GAOG;AACH,SAAgB,oBAAoB,CAChC,aAGE;IAEF,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IAExC,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC;QAC1C,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACvC,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAdD,oDAcC;AAED;;;;;;GAMG;AACH,SAAgB,sBAAsB,CAClC,WAAmB,EACnB,WAAgC;IAOhC,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACpF,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,WAAW,CAAC,CAAC;IAE9D,MAAM,SAAS,GAAG;QACd;YACI,SAAS,EAAE,iBAAiB;YAC5B,IAAI,EAAE,cAAc;YACpB,UAAU,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,GAAG,WAAW,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;SACzE;QACD,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;YACxD,SAAS,EAAE,WAAW,IAAI,EAAE;YAC5B,IAAI;YACJ,UAAU,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,WAAW,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;SAC/D,CAAC,CAAC;KACN,CAAC;IAEF,OAAO;QACH,cAAc;QACd,gBAAgB,EAAE,WAAW;QAC7B,KAAK,EAAE,WAAW;QAClB,SAAS;KACZ,CAAC;AACN,CAAC;AA/BD,wDA+BC;AAED;;;;;;GAMG;AACH,SAAgB,iBAAiB,CAAC,SAAiB,EAAE,cAAsB;IACvE,OAAO,SAAS,GAAG,cAAc,CAAC;AACtC,CAAC;AAFD,8CAEC;AAED;;;;;;;GAOG;AACH,SAAgB,uBAAuB,CACnC,SAAiB,EACjB,WAAmB,EACnB,UAAkB;IAMlB,MAAM,WAAW,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,MAAM,iBAAiB,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,MAAM,mBAAmB,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAExE,OAAO;QACH,WAAW;QACX,iBAAiB;QACjB,mBAAmB;KACtB,CAAC;AACN,CAAC;AAlBD,0DAkBC"}