@memberjunction/testing-engine 0.0.1 → 2.119.0

package/dist/oracles/SchemaValidatorOracle.js

@@ -0,0 +1,193 @@
+"use strict";
+/**
+ * @fileoverview Schema validation oracle implementation
+ * @module @memberjunction/testing-engine
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SchemaValidatorOracle = void 0;
+/**
+ * Schema Validator Oracle.
+ *
+ * Validates that actual output conforms to an expected JSON schema.
+ * Uses JSON Schema draft-07 specification for validation.
+ *
+ * Configuration:
+ * - schema: JSON Schema object defining expected structure
+ * - strict: Whether to fail on additional properties (default: false)
+ *
+ * @example
+ * ```typescript
+ * const oracle = new SchemaValidatorOracle();
+ * const result = await oracle.evaluate({
+ *     actualOutput: { name: 'John', age: 30 },
+ *     expectedOutput: {
+ *         responseSchema: {
+ *             type: 'object',
+ *             required: ['name', 'age'],
+ *             properties: {
+ *                 name: { type: 'string' },
+ *                 age: { type: 'number' }
+ *             }
+ *         }
+ *     }
+ * }, {});
+ * ```
+ */
+class SchemaValidatorOracle {
+    constructor() {
+        this.type = 'schema-validate';
+    }
+    /**
+     * Evaluate actual output against JSON schema.
+     *
+     * @param input - Oracle input with expected schema and actual output
+     * @param config - Oracle configuration
+     * @returns Oracle result with pass/fail and validation details
+     */
+    async evaluate(input, config) {
+        try {
+            // Extract schema from expected outcomes
+            const schema = input.expectedOutput?.responseSchema;
+            if (!schema) {
+                return {
+                    oracleType: this.type,
+                    passed: false,
+                    score: 0,
+                    message: 'No responseSchema provided in ExpectedOutcomes'
+                };
+            }
+            // Validate actual output against schema
+            const validationErrors = this.validateAgainstSchema(input.actualOutput, schema, config.strict);
+            if (validationErrors.length === 0) {
+                return {
+                    oracleType: this.type,
+                    passed: true,
+                    score: 1.0,
+                    message: 'Output matches expected schema'
+                };
+            }
+            else {
+                return {
+                    oracleType: this.type,
+                    passed: false,
+                    score: 0,
+                    message: `Schema validation failed: ${validationErrors.join(', ')}`,
+                    details: { validationErrors }
+                };
+            }
+        }
+        catch (error) {
+            return {
+                oracleType: this.type,
+                passed: false,
+                score: 0,
+                message: `Schema validation error: ${error.message}`
+            };
+        }
+    }
+    /**
+     * Validate data against JSON schema.
+     * @private
+     */
+    validateAgainstSchema(data, schema, strict = false) {
+        const errors = [];
+        // Simple JSON Schema validation implementation
+        // For production, consider using a library like ajv
+        this.validateValue(data, schema, 'root', errors, strict);
+        return errors;
+    }
+    /**
+     * Validate a single value against schema.
+     * @private
+     */
+    validateValue(value, schema, path, errors, strict) {
+        // Check type
+        const expectedType = schema.type;
+        const actualType = this.getType(value);
+        if (expectedType && actualType !== expectedType) {
+            errors.push(`${path}: Expected type ${expectedType}, got ${actualType}`);
+            return;
+        }
+        // Check required properties for objects
+        if (actualType === 'object' && value !== null) {
+            const obj = value;
+            const required = schema.required || [];
+            const properties = schema.properties || {};
+            // Check required fields
+            for (const requiredProp of required) {
+                if (!(requiredProp in obj)) {
+                    errors.push(`${path}: Missing required property '${requiredProp}'`);
+                }
+            }
+            // Validate properties
+            for (const [key, val] of Object.entries(obj)) {
+                if (properties[key]) {
+                    this.validateValue(val, properties[key], `${path}.${key}`, errors, strict);
+                }
+                else if (strict) {
+                    errors.push(`${path}: Unexpected property '${key}'`);
+                }
+            }
+        }
+        // Check array items
+        if (actualType === 'array' && schema.items) {
+            const arr = value;
+            const itemSchema = schema.items;
+            for (let i = 0; i < arr.length; i++) {
+                this.validateValue(arr[i], itemSchema, `${path}[${i}]`, errors, strict);
+            }
+        }
+        // Check enum values
+        if (schema.enum) {
+            const enumValues = schema.enum;
+            if (!enumValues.includes(value)) {
+                errors.push(`${path}: Value must be one of [${enumValues.join(', ')}], got '${value}'`);
+            }
+        }
+        // Check string patterns
+        if (actualType === 'string' && schema.pattern) {
+            const pattern = new RegExp(schema.pattern);
+            if (!pattern.test(value)) {
+                errors.push(`${path}: String does not match pattern ${schema.pattern}`);
+            }
+        }
+        // Check numeric constraints
+        if (actualType === 'number') {
+            const num = value;
+            if (schema.minimum != null && num < schema.minimum) {
+                errors.push(`${path}: Value ${num} is less than minimum ${schema.minimum}`);
+            }
+            if (schema.maximum != null && num > schema.maximum) {
+                errors.push(`${path}: Value ${num} is greater than maximum ${schema.maximum}`);
+            }
+        }
+    }
+    /**
+     * Get JSON Schema type for a value.
+     * @private
+     */
+    getType(value) {
+        if (value === null) {
+            return 'null';
+        }
+        if (Array.isArray(value)) {
+            return 'array';
+        }
+        const jsType = typeof value;
+        if (jsType === 'boolean') {
+            return 'boolean';
+        }
+        if (jsType === 'number') {
+            return Number.isInteger(value) ? 'integer' : 'number';
+        }
+        if (jsType === 'string') {
+            return 'string';
+        }
+        if (jsType === 'object') {
+            return 'object';
+        }
+        return 'unknown';
+    }
+}
+exports.SchemaValidatorOracle = SchemaValidatorOracle;
+//# sourceMappingURL=SchemaValidatorOracle.js.map

package/dist/oracles/SchemaValidatorOracle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SchemaValidatorOracle.js","sourceRoot":"","sources":["../../src/oracles/SchemaValidatorOracle.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAKH;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAa,qBAAqB;IAAlC;QACa,SAAI,GAAG,iBAAiB,CAAC;IAgMtC,CAAC;IA9LG;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAkB,EAAE,MAAoB;QACnD,IAAI,CAAC;YACD,wCAAwC;YACxC,MAAM,MAAM,GAAI,KAAK,CAAC,cAAsB,EAAE,cAAc,CAAC;YAC7D,IAAI,CAAC,MAAM,EAAE,CAAC;gBACV,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,gDAAgD;iBAC5D,CAAC;YACN,CAAC;YAED,wCAAwC;YACxC,MAAM,gBAAgB,GAAG,IAAI,CAAC,qBAAqB,CAC/C,KAAK,CAAC,YAAY,EAClB,MAAM,EACN,MAAM,CAAC,MAAiB,CAC3B,CAAC;YAEF,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAChC,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,IAAI;oBACZ,KAAK,EAAE,GAAG;oBACV,OAAO,EAAE,gCAAgC;iBAC5C,CAAC;YACN,CAAC;iBAAM,CAAC;gBACJ,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,6BAA6B,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;oBACnE,OAAO,EAAE,EAAE,gBAAgB,EAAE;iBAChC,CAAC;YACN,CAAC;QAEL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO;gBACH,UAAU,EAAE,IAAI,CAAC,IAAI;gBACrB,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,4BAA6B,KAAe,CAAC,OAAO,EAAE;aAClE,CAAC;QACN,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,qBAAqB,CACzB,IAAa,EACb,MAA+B,EAC/B,SAAkB,KAAK;QAEvB,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,+CAA+C;QAC/C,oDAAoD;QACpD,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAEzD,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;OAGG;IACK,aAAa,CACjB,KAAc,EACd,MAA+B,EAC/B,IAAY,EACZ,MAAgB,EAChB,MAAe;QAEf,aAAa;QACb,MAAM,YAAY,GAAG,MAAM,CAAC,IAAc,CAAC;QAC3C,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAEvC,IAAI,YAAY,IAAI,UAAU,KAAK,YAAY,EAAE,CAAC;YAC9C,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,mBAAmB,YAAY,SAAS,UAAU,EAAE,CAAC,CAAC;YACzE,OAAO;QACX,CAAC;QAED,wCAAwC;QACxC,IAAI,UAAU,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YAC5C,MAAM,GAAG,GAAG,KAAgC,CAAC;YAC7C,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAoB,IAAI,EAAE,CAAC;YACnD,MAAM,UAAU,GAAG,MAAM,CAAC,UAAqD,IAAI,EAAE,CAAC;YAEtF,wBAAwB;YACxB,KAAK,MAAM,YAAY,IAAI,QAAQ,EAAE,CAAC;gBAClC,IAAI,CAAC,CAAC,YAAY,IAAI,GAAG,CAAC,EAAE,CAAC;oBACzB,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,gCAAgC,YAAY,GAAG,CAAC,CAAC;gBACxE,CAAC;YACL,CAAC;YAED,sBAAsB;YACtB,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3C,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClB,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,IAAI,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;gBAC/E,CAAC;qBAAM,IAAI,MAAM,EAAE,CAAC;oBAChB,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,0BAA0B,GAAG,GAAG,CAAC,CAAC;gBACzD,CAAC;YACL,CAAC;QACL,CAAC;QAED,oBAAoB;QACpB,IAAI,UAAU,KAAK,OAAO,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YACzC,MAAM,GAAG,GAAG,KAAkB,CAAC;YAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,KAAgC,CAAC;YAE3D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAClC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;YAC5E,CAAC;QACL,CAAC;QAED,oBAAoB;QACpB,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YACd,MAAM,UAAU,GAAG,MAAM,CAAC,IAAiB,CAAC;YAC5C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,MAAM,CAAC,IAAI,CACP,GAAG,IAAI,2BAA2B,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,KAAK,GAAG,CAC7E,CAAC;YACN,CAAC;QACL,CAAC;QAED,wBAAwB;QACxB,IAAI,UAAU,KAAK,QAAQ,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YAC5C,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,OAAiB,CAAC,CAAC;YACrD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAe,CAAC,EAAE,CAAC;gBACjC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,mCAAmC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;YAC5E,CAAC;QACL,CAAC;QAED,4BAA4B;QAC5B,IAAI,UAAU,KAAK,QAAQ,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,KAAe,CAAC;YAE5B,IAAI,MAAM,CAAC,OAAO,IAAI,IAAI,IAAI,GAAG,GAAI,MAAM,CAAC,OAAkB,EAAE,CAAC;gBAC7D,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,WAAW,GAAG,yBAAyB,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;YAChF,CAAC;YAED,IAAI,MAAM,CAAC,OAAO,IAAI,IAAI,IAAI,GAAG,GAAI,MAAM,CAAC,OAAkB,EAAE,CAAC;gBAC7D,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,WAAW,GAAG,4BAA4B,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;YACnF,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,OAAO,CAAC,KAAc;QAC1B,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,OAAO,MAAM,CAAC;QAClB,CAAC;QAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,OAAO,CAAC;QACnB,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,KAAK,CAAC;QAE5B,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACvB,OAAO,SAAS,CAAC;QACrB,CAAC;QAED,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;YACtB,OAAO,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC1D,CAAC;QAED,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;YACtB,OAAO,QAAQ,CAAC;QACpB,CAAC;QAED,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;YACtB,OAAO,QAAQ,CAAC;QACpB,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;CACJ;AAjMD,sDAiMC"}

package/dist/oracles/TraceValidatorOracle.d.ts

@@ -0,0 +1,41 @@
+/**
+ * @fileoverview Trace validation oracle implementation
+ * @module @memberjunction/testing-engine
+ */
+import { IOracle } from './IOracle';
+import { OracleInput, OracleConfig, OracleResult } from '../types';
+/**
+ * Trace Validator Oracle.
+ *
+ * Validates that agent execution trace has no errors.
+ * Checks AgentRun status and all AgentRunStep records for error conditions.
+ *
+ * Configuration:
+ * - allowWarnings: Whether to allow warning-level issues (default: true)
+ * - requiredSteps: Minimum number of steps expected (optional)
+ * - maxSteps: Maximum number of steps allowed (optional)
+ *
+ * @example
+ * ```typescript
+ * const oracle = new TraceValidatorOracle();
+ * const result = await oracle.evaluate({
+ *     targetEntity: agentRun,
+ *     contextUser
+ * }, {
+ *     allowWarnings: true,
+ *     requiredSteps: 2
+ * });
+ * ```
+ */
+export declare class TraceValidatorOracle implements IOracle {
+    readonly type = "trace-no-errors";
+    /**
+     * Evaluate agent run trace for errors.
+     *
+     * @param input - Oracle input with agent run entity
+     * @param config - Oracle configuration
+     * @returns Oracle result with pass/fail and trace analysis
+     */
+    evaluate(input: OracleInput, config: OracleConfig): Promise<OracleResult>;
+}
+//# sourceMappingURL=TraceValidatorOracle.d.ts.map

package/dist/oracles/TraceValidatorOracle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TraceValidatorOracle.d.ts","sourceRoot":"","sources":["../../src/oracles/TraceValidatorOracle.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAEnE;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,qBAAa,oBAAqB,YAAW,OAAO;IAChD,QAAQ,CAAC,IAAI,qBAAqB;IAElC;;;;;;OAMG;IACG,QAAQ,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;CA+HlF"}

package/dist/oracles/TraceValidatorOracle.js

@@ -0,0 +1,159 @@
+"use strict";
+/**
+ * @fileoverview Trace validation oracle implementation
+ * @module @memberjunction/testing-engine
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TraceValidatorOracle = void 0;
+const core_1 = require("@memberjunction/core");
+/**
+ * Trace Validator Oracle.
+ *
+ * Validates that agent execution trace has no errors.
+ * Checks AgentRun status and all AgentRunStep records for error conditions.
+ *
+ * Configuration:
+ * - allowWarnings: Whether to allow warning-level issues (default: true)
+ * - requiredSteps: Minimum number of steps expected (optional)
+ * - maxSteps: Maximum number of steps allowed (optional)
+ *
+ * @example
+ * ```typescript
+ * const oracle = new TraceValidatorOracle();
+ * const result = await oracle.evaluate({
+ *     targetEntity: agentRun,
+ *     contextUser
+ * }, {
+ *     allowWarnings: true,
+ *     requiredSteps: 2
+ * });
+ * ```
+ */
+class TraceValidatorOracle {
+    constructor() {
+        this.type = 'trace-no-errors';
+    }
+    /**
+     * Evaluate agent run trace for errors.
+     *
+     * @param input - Oracle input with agent run entity
+     * @param config - Oracle configuration
+     * @returns Oracle result with pass/fail and trace analysis
+     */
+    async evaluate(input, config) {
+        try {
+            // Get agent run entity
+            const agentRun = input.targetEntity;
+            if (!agentRun) {
+                return {
+                    oracleType: this.type,
+                    passed: false,
+                    score: 0,
+                    message: 'No agent run entity provided'
+                };
+            }
+            // Check agent run status
+            if (agentRun.Status === 'Failed' || agentRun.Status === 'Cancelled') {
+                return {
+                    oracleType: this.type,
+                    passed: false,
+                    score: 0,
+                    message: `Agent run failed with status: ${agentRun.Status}`,
+                    details: {
+                        agentRunId: agentRun.ID,
+                        status: agentRun.Status
+                    }
+                };
+            }
+            // Load agent run steps
+            const rv = new core_1.RunView();
+            const stepsResult = await rv.RunView({
+                EntityName: 'MJ: AI Agent Run Steps',
+                ExtraFilter: `AgentRunID='${agentRun.ID}'`,
+                OrderBy: 'Sequence ASC',
+                ResultType: 'entity_object'
+            }, input.contextUser);
+            if (!stepsResult.Success) {
+                return {
+                    oracleType: this.type,
+                    passed: false,
+                    score: 0,
+                    message: `Failed to load agent run steps: ${stepsResult.ErrorMessage}`
+                };
+            }
+            const steps = stepsResult.Results || [];
+            // Check step count constraints
+            const requiredSteps = config.requiredSteps;
+            const maxSteps = config.maxSteps;
+            if (requiredSteps && steps.length < requiredSteps) {
+                return {
+                    oracleType: this.type,
+                    passed: false,
+                    score: 0,
+                    message: `Expected at least ${requiredSteps} steps, got ${steps.length}`,
+                    details: { stepCount: steps.length, requiredSteps }
+                };
+            }
+            if (maxSteps && steps.length > maxSteps) {
+                return {
+                    oracleType: this.type,
+                    passed: false,
+                    score: 0,
+                    message: `Expected at most ${maxSteps} steps, got ${steps.length}`,
+                    details: { stepCount: steps.length, maxSteps }
+                };
+            }
+            // Check each step for errors
+            const allowWarnings = config.allowWarnings !== false; // Default to true
+            const errors = [];
+            const warnings = [];
+            for (const step of steps) {
+                if (step.Status === 'Failed') {
+                    errors.push(`Step ${step.StepNumber}: ${step.StepName} - ${step.Status}`);
+                }
+                // Note: Output field doesn't exist on AIAgentRunStepEntity
+                // If we need to check output, we'd need to load it from another source
+            }
+            // Determine result
+            if (errors.length > 0) {
+                return {
+                    oracleType: this.type,
+                    passed: false,
+                    score: 0,
+                    message: `Trace contains ${errors.length} error(s)`,
+                    details: { errors, warnings, stepCount: steps.length }
+                };
+            }
+            if (!allowWarnings && warnings.length > 0) {
+                return {
+                    oracleType: this.type,
+                    passed: false,
+                    score: 0.5,
+                    message: `Trace contains ${warnings.length} warning(s)`,
+                    details: { warnings, stepCount: steps.length }
+                };
+            }
+            // Success
+            return {
+                oracleType: this.type,
+                passed: true,
+                score: 1.0,
+                message: `Trace clean with ${steps.length} step(s)`,
+                details: {
+                    stepCount: steps.length,
+                    warnings: warnings.length > 0 ? warnings : undefined
+                }
+            };
+        }
+        catch (error) {
+            return {
+                oracleType: this.type,
+                passed: false,
+                score: 0,
+                message: `Trace validation error: ${error.message}`
+            };
+        }
+    }
+}
+exports.TraceValidatorOracle = TraceValidatorOracle;
+//# sourceMappingURL=TraceValidatorOracle.js.map

package/dist/oracles/TraceValidatorOracle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TraceValidatorOracle.js","sourceRoot":"","sources":["../../src/oracles/TraceValidatorOracle.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,+CAA+C;AAK/C;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAa,oBAAoB;IAAjC;QACa,SAAI,GAAG,iBAAiB,CAAC;IAwItC,CAAC;IAtIG;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAkB,EAAE,MAAoB;QACnD,IAAI,CAAC;YACD,uBAAuB;YACvB,MAAM,QAAQ,GAAG,KAAK,CAAC,YAAgC,CAAC;YACxD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACZ,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,8BAA8B;iBAC1C,CAAC;YACN,CAAC;YAED,yBAAyB;YACzB,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClE,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,iCAAiC,QAAQ,CAAC,MAAM,EAAE;oBAC3D,OAAO,EAAE;wBACL,UAAU,EAAE,QAAQ,CAAC,EAAE;wBACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;qBAC1B;iBACJ,CAAC;YACN,CAAC;YAED,uBAAuB;YACvB,MAAM,EAAE,GAAG,IAAI,cAAO,EAAE,CAAC;YACzB,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,OAAO,CAAuB;gBACvD,UAAU,EAAE,wBAAwB;gBACpC,WAAW,EAAE,eAAe,QAAQ,CAAC,EAAE,GAAG;gBAC1C,OAAO,EAAE,cAAc;gBACvB,UAAU,EAAE,eAAe;aAC9B,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;YAEtB,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;gBACvB,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,mCAAmC,WAAW,CAAC,YAAY,EAAE;iBACzE,CAAC;YACN,CAAC;YAED,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,IAAI,EAAE,CAAC;YAExC,+BAA+B;YAC/B,MAAM,aAAa,GAAG,MAAM,CAAC,aAAuB,CAAC;YACrD,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAkB,CAAC;YAE3C,IAAI,aAAa,IAAI,KAAK,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;gBAChD,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,qBAAqB,aAAa,eAAe,KAAK,CAAC,MAAM,EAAE;oBACxE,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,MAAM,EAAE,aAAa,EAAE;iBACtD,CAAC;YACN,CAAC;YAED,IAAI,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;gBACtC,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,oBAAoB,QAAQ,eAAe,KAAK,CAAC,MAAM,EAAE;oBAClE,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE;iBACjD,CAAC;YACN,CAAC;YAED,6BAA6B;YAC7B,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,KAAK,KAAK,CAAC,CAAC,kBAAkB;YACxE,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAa,EAAE,CAAC;YAE9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACvB,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;oBAC3B,MAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,UAAU,KAAK,IAAI,CAAC,QAAQ,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC9E,CAAC;gBAED,2DAA2D;gBAC3D,uEAAuE;YAC3E,CAAC;YAED,mBAAmB;YACnB,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpB,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,kBAAkB,MAAM,CAAC,MAAM,WAAW;oBACnD,OAAO,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,MAAM,EAAE;iBACzD,CAAC;YACN,CAAC;YAED,IAAI,CAAC,aAAa,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxC,OAAO;oBACH,UAAU,EAAE,IAAI,CAAC,IAAI;oBACrB,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,GAAG;oBACV,OAAO,EAAE,kBAAkB,QAAQ,CAAC,MAAM,aAAa;oBACvD,OAAO,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,MAAM,EAAE;iBACjD,CAAC;YACN,CAAC;YAED,UAAU;YACV,OAAO;gBACH,UAAU,EAAE,IAAI,CAAC,IAAI;gBACrB,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,GAAG;gBACV,OAAO,EAAE,oBAAoB,KAAK,CAAC,MAAM,UAAU;gBACnD,OAAO,EAAE;oBACL,SAAS,EAAE,KAAK,CAAC,MAAM;oBACvB,QAAQ,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;iBACvD;aACJ,CAAC;QAEN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO;gBACH,UAAU,EAAE,IAAI,CAAC,IAAI;gBACrB,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,2BAA4B,KAAe,CAAC,OAAO,EAAE;aACjE,CAAC;QACN,CAAC;IACL,CAAC;CACJ;AAzID,oDAyIC"}

package/dist/oracles/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Oracle implementations for test evaluation
+ */
+export * from './IOracle';
+export * from './SchemaValidatorOracle';
+export * from './TraceValidatorOracle';
+export * from './LLMJudgeOracle';
+export * from './ExactMatchOracle';
+export * from './SQLValidatorOracle';
+//# sourceMappingURL=index.d.ts.map

package/dist/oracles/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/oracles/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,cAAc,WAAW,CAAC;AAC1B,cAAc,yBAAyB,CAAC;AACxC,cAAc,wBAAwB,CAAC;AACvC,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC;AACnC,cAAc,sBAAsB,CAAC"}

package/dist/oracles/index.js

@@ -0,0 +1,26 @@
+"use strict";
+/**
+ * Oracle implementations for test evaluation
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./IOracle"), exports);
+__exportStar(require("./SchemaValidatorOracle"), exports);
+__exportStar(require("./TraceValidatorOracle"), exports);
+__exportStar(require("./LLMJudgeOracle"), exports);
+__exportStar(require("./ExactMatchOracle"), exports);
+__exportStar(require("./SQLValidatorOracle"), exports);
+//# sourceMappingURL=index.js.map

package/dist/oracles/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/oracles/index.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;AAEH,4CAA0B;AAC1B,0DAAwC;AACxC,yDAAuC;AACvC,mDAAiC;AACjC,qDAAmC;AACnC,uDAAqC"}