@memberjunction/testing-engine-base 2.129.0 → 2.130.1

package/dist/index.d.ts

@@ -4,4 +4,5 @@
  * Metadata cache for test framework (UI-safe, no execution logic)
  */
 export * from './TestEngineBase';
+export * from './types';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,kBAAkB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,kBAAkB,CAAC;AACjC,cAAc,SAAS,CAAC"}

package/dist/index.js

@@ -20,4 +20,5 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./TestEngineBase"), exports);
+__exportStar(require("./types"), exports);
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;;GAIG;;;;;;;;;;;;;;;;AAEH,mDAAiC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;;GAIG;;;;;;;;;;;;;;;;AAEH,mDAAiC;AACjC,0CAAwB"}

package/dist/types.d.ts

@@ -0,0 +1,421 @@
+/**
+ * Core type definitions for the Testing Framework
+ * These types are UI-safe and do not depend on execution logic
+ */
+/**
+ * Log message from test execution
+ */
+export interface TestLogMessage {
+    /**
+     * Timestamp when the message was logged
+     */
+    timestamp: Date;
+    /**
+     * Log level
+     */
+    level: 'info' | 'warn' | 'error' | 'debug';
+    /**
+     * Log message content
+     */
+    message: string;
+    /**
+     * Optional metadata for additional context
+     */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * Progress callback for test execution
+ */
+export interface TestProgress {
+    /**
+     * Current execution step
+     */
+    step: string;
+    /**
+     * Progress percentage (0-100)
+     */
+    percentage: number;
+    /**
+     * Human-readable message
+     */
+    message: string;
+    /**
+     * Additional metadata
+     */
+    metadata?: {
+        testName?: string;
+        testRun?: unknown;
+        driverType?: string;
+        oracleType?: string;
+        [key: string]: unknown;
+    };
+}
+/**
+ * Options for running a single test
+ */
+export interface TestRunOptions {
+    /**
+     * Verbose logging
+     */
+    verbose?: boolean;
+    /**
+     * Validate configuration without executing
+     */
+    dryRun?: boolean;
+    /**
+     * Environment context (dev, staging, prod)
+     */
+    environment?: string;
+    /**
+     * Git commit SHA for versioning
+     */
+    gitCommit?: string;
+    /**
+     * Agent/system version being tested
+     */
+    agentVersion?: string;
+    /**
+     * Override test configuration
+     */
+    configOverride?: Record<string, unknown>;
+    /**
+     * Progress callback for real-time updates
+     */
+    progressCallback?: (progress: TestProgress) => void;
+    /**
+     * Log callback for streaming execution details to the test run log
+     */
+    logCallback?: (message: TestLogMessage) => void;
+    /**
+     * Tags to apply to the test run (JSON string array)
+     */
+    tags?: string;
+}
+/**
+ * Options for running a test suite
+ */
+export interface SuiteRunOptions extends TestRunOptions {
+    /**
+     * Run tests in parallel
+     */
+    parallel?: boolean;
+    /**
+     * Stop on first failure
+     */
+    failFast?: boolean;
+    /**
+     * Maximum parallel tests (if parallel=true)
+     */
+    maxParallel?: number;
+    /**
+     * Run only specific sequence numbers (e.g., [1, 3, 5] runs tests at those positions)
+     */
+    sequence?: number[];
+    /**
+     * Run only specific tests by their IDs.
+     * If provided, only tests with matching IDs will be executed.
+     */
+    selectedTestIds?: string[];
+    /**
+     * Start execution from this sequence number (inclusive).
+     * Tests with sequence numbers less than this value will be skipped.
+     */
+    sequenceStart?: number;
+    /**
+     * Stop execution at this sequence number (inclusive).
+     * Tests with sequence numbers greater than this value will be skipped.
+     */
+    sequenceEnd?: number;
+}
+/**
+ * Oracle evaluation result
+ */
+export interface OracleResult {
+    /**
+     * Oracle type that produced this result
+     */
+    oracleType: string;
+    /**
+     * Whether the oracle check passed
+     */
+    passed: boolean;
+    /**
+     * Numeric score (0.0 to 1.0)
+     */
+    score: number;
+    /**
+     * Human-readable message
+     */
+    message: string;
+    /**
+     * Additional details (oracle-specific)
+     */
+    details?: unknown;
+}
+/**
+ * Result from running a single test
+ */
+export interface TestRunResult {
+    /**
+     * Test Run ID
+     */
+    testRunId: string;
+    /**
+     * Test ID
+     */
+    testId: string;
+    /**
+     * Test name (from lookup field)
+     */
+    testName: string;
+    /**
+     * Test execution status
+     */
+    status: 'Passed' | 'Failed' | 'Skipped' | 'Error' | 'Timeout';
+    /**
+     * Overall score (0.0000 to 1.0000)
+     */
+    score: number;
+    /**
+     * Number of checks that passed
+     */
+    passedChecks: number;
+    /**
+     * Number of checks that failed
+     */
+    failedChecks: number;
+    /**
+     * Total number of checks
+     */
+    totalChecks: number;
+    /**
+     * Oracle evaluation results
+     */
+    oracleResults: OracleResult[];
+    /**
+     * Optional sub-category or variant label for the test target.
+     * Use for ad-hoc labeling or to distinguish test scenarios within the same entity type.
+     * Examples: "Summarization", "Classification", "Code Review", "Multi-turn Chat"
+     */
+    targetType: string;
+    /**
+     * Entity ID identifying the type of target being tested.
+     * References Entity.ID (e.g., Entity ID for "MJ: AI Agent Runs").
+     */
+    targetLogEntityId?: string;
+    /**
+     * Target entity ID (e.g., AIAgentRun.ID)
+     */
+    targetLogId: string;
+    /**
+     * Execution duration in milliseconds
+     */
+    durationMs: number;
+    /**
+     * Cost in USD
+     */
+    totalCost: number;
+    /**
+     * When execution started
+     */
+    startedAt: Date;
+    /**
+     * When execution completed
+     */
+    completedAt: Date;
+    /**
+     * Error message if status is Error
+     */
+    errorMessage?: string;
+    /**
+     * Iteration number for repeated tests (when RepeatCount > 1)
+     */
+    sequence?: number;
+}
+/**
+ * Result from running a test suite
+ */
+export interface TestSuiteRunResult {
+    /**
+     * Suite Run ID
+     */
+    suiteRunId: string;
+    /**
+     * Suite ID
+     */
+    suiteId: string;
+    /**
+     * Suite name (from lookup field)
+     */
+    suiteName: string;
+    /**
+     * Suite execution status
+     */
+    status: 'Completed' | 'Failed' | 'Cancelled' | 'Pending' | 'Running';
+    /**
+     * Tests that passed
+     */
+    passedTests: number;
+    /**
+     * Tests that failed
+     */
+    failedTests: number;
+    /**
+     * Total tests
+     */
+    totalTests: number;
+    /**
+     * Average score across all tests
+     */
+    averageScore: number;
+    /**
+     * Individual test results
+     */
+    testResults: TestRunResult[];
+    /**
+     * Total duration in milliseconds
+     */
+    durationMs: number;
+    /**
+     * Total cost in USD
+     */
+    totalCost: number;
+    /**
+     * When execution started
+     */
+    startedAt: Date;
+    /**
+     * When execution completed
+     */
+    completedAt: Date;
+}
+/**
+ * Scoring weights for different evaluation dimensions
+ */
+export interface ScoringWeights {
+    /**
+     * Weight for each oracle type
+     * Keys are oracle types, values are weights (should sum to 1.0)
+     */
+    [oracleType: string]: number;
+}
+/**
+ * Validation result for test configuration
+ */
+export interface ValidationResult {
+    /**
+     * Whether validation passed
+     */
+    valid: boolean;
+    /**
+     * Validation errors (blocking issues)
+     */
+    errors: ValidationError[];
+    /**
+     * Validation warnings (non-blocking issues)
+     */
+    warnings: ValidationWarning[];
+}
+/**
+ * Validation error
+ */
+export interface ValidationError {
+    /**
+     * Error category
+     */
+    category: 'configuration' | 'input' | 'expected-outcome';
+    /**
+     * Error message
+     */
+    message: string;
+    /**
+     * Field path (if applicable)
+     */
+    field?: string;
+    /**
+     * Suggested fix
+     */
+    suggestion?: string;
+}
+/**
+ * Validation warning
+ */
+export interface ValidationWarning {
+    /**
+     * Warning category
+     */
+    category: 'best-practice' | 'performance' | 'cost';
+    /**
+     * Warning message
+     */
+    message: string;
+    /**
+     * Recommendation
+     */
+    recommendation?: string;
+}
+/**
+ * Execution context details for test runs.
+ * Stored as JSON in the RunContextDetails field of TestRun and TestSuiteRun entities.
+ * Enables cross-server aggregation and detailed environment tracking.
+ */
+export interface RunContextDetails {
+    /**
+     * Operating system type (e.g., "darwin", "linux", "win32")
+     */
+    osType?: string;
+    /**
+     * Operating system version/release
+     */
+    osVersion?: string;
+    /**
+     * Node.js version used to run the tests
+     */
+    nodeVersion?: string;
+    /**
+     * Timezone identifier (e.g., "America/New_York", "UTC")
+     */
+    timezone?: string;
+    /**
+     * System locale (e.g., "en-US", "fr-FR")
+     */
+    locale?: string;
+    /**
+     * IP address of the machine running tests (useful for network debugging)
+     */
+    ipAddress?: string;
+    /**
+     * CI/CD provider name (e.g., "GitHub Actions", "Azure DevOps", "Jenkins")
+     */
+    ciProvider?: string;
+    /**
+     * CI/CD pipeline or workflow ID
+     */
+    pipelineId?: string;
+    /**
+     * Build number or run number from CI/CD
+     */
+    buildNumber?: string;
+    /**
+     * Git branch name
+     */
+    branch?: string;
+    /**
+     * Pull request number (if applicable)
+     */
+    prNumber?: string;
+    /**
+     * Additional custom properties for extensibility
+     */
+    [key: string]: unknown;
+}
+/**
+ * Oracle configuration (can have any additional properties)
+ */
+export interface OracleConfig {
+    /**
+     * Oracle-specific configuration properties
+     */
+    [key: string]: unknown;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;OAEG;IACH,SAAS,EAAE,IAAI,CAAC;IAEhB;;OAEG;IACH,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,OAAO,CAAC;IAE3C;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,QAAQ,CAAC,EAAE;QACT,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACxB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IAEjB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEzC;;OAEG;IACH,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;IAEpD;;OAEG;IACH,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,cAAc,KAAK,IAAI,CAAC;IAEhD;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,eAAgB,SAAQ,cAAc;IACrD;;OAEG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;OAEG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IAEpB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAE3B;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,MAAM,EAAE,OAAO,CAAC;IAEhB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,SAAS,CAAC;IAE9D;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,aAAa,EAAE,YAAY,EAAE,CAAC;IAE9B;;;;OAIG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,SAAS,EAAE,IAAI,CAAC;IAEhB;;OAEG;IACH,WAAW,EAAE,IAAI,CAAC;IAElB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,MAAM,EAAE,WAAW,GAAG,QAAQ,GAAG,WAAW,GAAG,SAAS,GAAG,SAAS,CAAC;IAErE;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,WAAW,EAAE,aAAa,EAAE,CAAC;IAE7B;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,SAAS,EAAE,IAAI,CAAC;IAEhB;;OAEG;IACH,WAAW,EAAE,IAAI,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;OAEG;IACH,KAAK,EAAE,OAAO,CAAC;IAEf;;OAEG;IACH,MAAM,EAAE,eAAe,EAAE,CAAC;IAE1B;;OAEG;IACH,QAAQ,EAAE,iBAAiB,EAAE,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;OAEG;IACH,QAAQ,EAAE,eAAe,GAAG,OAAO,GAAG,kBAAkB,CAAC;IAEzD;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,QAAQ,EAAE,eAAe,GAAG,aAAa,GAAG,MAAM,CAAC;IAEnD;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B;;OAEG;IACH,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB"}

package/dist/types.js

@@ -0,0 +1,7 @@
+"use strict";
+/**
+ * Core type definitions for the Testing Framework
+ * These types are UI-safe and do not depend on execution logic
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":";AAAA;;;GAGG"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@memberjunction/testing-engine-base",
-  "version": "2.129.0",
+  "version": "2.130.1",
   "description": "MemberJunction Testing Framework Engine Base - Metadata cache for test types, suites, and tests (UI-safe, no execution)",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -20,9 +20,9 @@
     "@types/debug": "^4.1.12"
   },
   "dependencies": {
-    "@memberjunction/global": "2.129.0",
-    "@memberjunction/core": "2.129.0",
-    "@memberjunction/core-entities": "2.129.0",
+    "@memberjunction/global": "2.130.1",
+    "@memberjunction/core": "2.130.1",
+    "@memberjunction/core-entities": "2.130.1",
     "rxjs": "^7.8.1",
     "debug": "^4.4.0"
   },