@roarkanalytics/sdk 0.332.0 → 2.0.0

package/resources/evaluation.d.ts

@@ -8,13 +8,13 @@ export declare class Evaluation extends APIResource {
     /**
      * Returns a specific evaluator with its blocks and configuration.
      */
-    getEvaluatorById(evaluatorId: string, options?: Core.RequestOptions): Core.APIPromise<unknown>;
+    getEvaluatorById(evaluatorId: string, options?: Core.RequestOptions): Core.APIPromise<EvaluationGetEvaluatorByIDResponse>;
     /**
      * Returns a list of evaluators with their blocks and configuration for the
      * authenticated project.
      */
-    getEvaluators(query?: EvaluationGetEvaluatorsParams, options?: Core.RequestOptions): Core.APIPromise<unknown>;
-    getEvaluators(options?: Core.RequestOptions): Core.APIPromise<unknown>;
+    getEvaluators(query?: EvaluationGetEvaluatorsParams, options?: Core.RequestOptions): Core.APIPromise<EvaluationGetEvaluatorsResponse>;
+    getEvaluators(options?: Core.RequestOptions): Core.APIPromise<EvaluationGetEvaluatorsResponse>;
     /**
      * Retrieve details of a specific evaluation job
      */
@@ -40,8 +40,752 @@ export declare namespace EvaluationCreateJobResponse {
         status: 'PENDING' | 'PROCESSING' | 'SUCCESS' | 'FAILURE';
     }
 }
-export type EvaluationGetEvaluatorByIDResponse = unknown;
-export type EvaluationGetEvaluatorsResponse = unknown;
+/**
+ * Evaluator with its configured blocks
+ */
+export interface EvaluationGetEvaluatorByIDResponse {
+    /**
+     * Unique identifier for the evaluator
+     */
+    id: string;
+    /**
+     * Array of evaluation blocks configured for this evaluator
+     */
+    blocks: Array<EvaluationGetEvaluatorByIDResponse.UnionMember0 | EvaluationGetEvaluatorByIDResponse.UnionMember1 | EvaluationGetEvaluatorByIDResponse.UnionMember2 | EvaluationGetEvaluatorByIDResponse.UnionMember3 | EvaluationGetEvaluatorByIDResponse.UnionMember4 | EvaluationGetEvaluatorByIDResponse.UnionMember5 | EvaluationGetEvaluatorByIDResponse.UnionMember6 | EvaluationGetEvaluatorByIDResponse.UnionMember7 | EvaluationGetEvaluatorByIDResponse.UnionMember8>;
+    /**
+     * ISO timestamp when the evaluator was created
+     */
+    createdAt: string;
+    /**
+     * Optional description of the evaluator
+     */
+    description: string | null;
+    /**
+     * Name of the evaluator
+     */
+    name: string;
+    /**
+     * Unique slug identifier for the evaluator
+     */
+    slug: string;
+    /**
+     * ISO timestamp when the evaluator was last updated
+     */
+    updatedAt: string;
+}
+export declare namespace EvaluationGetEvaluatorByIDResponse {
+    interface UnionMember0 {
+        /**
+         * Unique identifier for the block
+         */
+        id: string;
+        /**
+         * Block type identifier
+         */
+        blockType: 'CUSTOM_PROMPT';
+        /**
+         * Optional description of what this block evaluates
+         */
+        description: string | null;
+        /**
+         * Name of the metric this prompt evaluates
+         */
+        metricName: string;
+        /**
+         * Display name of the evaluation block
+         */
+        name: string;
+        /**
+         * Order in which this block is executed
+         */
+        orderIndex: number;
+        /**
+         * The prompt to evaluate the call against
+         */
+        prompt: string;
+        /**
+         * Minimum score threshold to pass evaluation (0-1)
+         */
+        threshold: number;
+        /**
+         * Weight of this block in the overall evaluation score (0-100)
+         */
+        weight: number;
+    }
+    interface UnionMember1 {
+        /**
+         * Unique identifier for the block
+         */
+        id: string;
+        /**
+         * Block type identifier
+         */
+        blockType: 'DATAFIELD_CHECK';
+        /**
+         * Optional description of what this block evaluates
+         */
+        description: string | null;
+        /**
+         * Criteria for evaluating the property
+         */
+        evaluationCriteria: string;
+        /**
+         * Whether this property must be present
+         */
+        isRequired: boolean;
+        /**
+         * Display name of the evaluation block
+         */
+        name: string;
+        /**
+         * Order in which this block is executed
+         */
+        orderIndex: number;
+        /**
+         * Name of the property to check
+         */
+        propertyName: string;
+        /**
+         * Minimum score threshold to pass evaluation (0-1)
+         */
+        threshold: number;
+        /**
+         * Expected type of the property value
+         */
+        valueType: string;
+        /**
+         * Weight of this block in the overall evaluation score (0-100)
+         */
+        weight: number;
+    }
+    interface UnionMember2 {
+        /**
+         * Unique identifier for the block
+         */
+        id: string;
+        /**
+         * Block type identifier
+         */
+        blockType: 'EMOTION';
+        /**
+         * Optional description of what this block evaluates
+         */
+        description: string | null;
+        /**
+         * Display name of the evaluation block
+         */
+        name: string;
+        /**
+         * Order in which this block is executed
+         */
+        orderIndex: number;
+        /**
+         * The emotion to detect (e.g., "joy", "anger", "sadness")
+         */
+        selectedEmotion: string;
+        /**
+         * Minimum confidence threshold for emotion detection (0-1)
+         */
+        threshold: number;
+        /**
+         * Weight of this block in the overall evaluation score (0-100)
+         */
+        weight: number;
+    }
+    interface UnionMember3 {
+        /**
+         * Unique identifier for the block
+         */
+        id: string;
+        /**
+         * Block type identifier
+         */
+        blockType: 'LATENCY';
+        /**
+         * Optional description of what this block evaluates
+         */
+        description: string | null;
+        /**
+         * Maximum number of silence periods allowed
+         */
+        maxAllowedSilences: number;
+        /**
+         * Minimum duration of silence in milliseconds to be considered
+         */
+        minSilenceDuration: number;
+        /**
+         * Display name of the evaluation block
+         */
+        name: string;
+        /**
+         * Order in which this block is executed
+         */
+        orderIndex: number;
+        /**
+         * Maximum allowed latency score
+         */
+        threshold: number;
+        /**
+         * Weight of this block in the overall evaluation score (0-100)
+         */
+        weight: number;
+    }
+    interface UnionMember4 {
+        /**
+         * Unique identifier for the block
+         */
+        id: string;
+        /**
+         * Block type identifier
+         */
+        blockType: 'POLITENESS';
+        /**
+         * Optional description of what this block evaluates
+         */
+        description: string | null;
+        /**
+         * Display name of the evaluation block
+         */
+        name: string;
+        /**
+         * Order in which this block is executed
+         */
+        orderIndex: number;
+        /**
+         * Minimum politeness score threshold (0-1)
+         */
+        threshold: number;
+        /**
+         * Weight of this block in the overall evaluation score (0-100)
+         */
+        weight: number;
+    }
+    interface UnionMember5 {
+        /**
+         * Unique identifier for the block
+         */
+        id: string;
+        /**
+         * Block type identifier
+         */
+        blockType: 'SENTIMENT';
+        /**
+         * Optional description of what this block evaluates
+         */
+        description: string | null;
+        /**
+         * Display name of the evaluation block
+         */
+        name: string;
+        /**
+         * Order in which this block is executed
+         */
+        orderIndex: number;
+        /**
+         * Minimum sentiment score threshold (0-1)
+         */
+        threshold: number;
+        /**
+         * Weight of this block in the overall evaluation score (0-100)
+         */
+        weight: number;
+    }
+    interface UnionMember6 {
+        /**
+         * Unique identifier for the block
+         */
+        id: string;
+        /**
+         * Block type identifier
+         */
+        blockType: 'TOOL_CALLS';
+        /**
+         * Optional description of what this block evaluates
+         */
+        description: string | null;
+        /**
+         * Condition that must be met for tool invocation
+         */
+        invocationCondition: string | null;
+        /**
+         * Minimum number of times the tool should be invoked
+         */
+        minInvocationCount: number | null;
+        /**
+         * Display name of the evaluation block
+         */
+        name: string;
+        /**
+         * Order in which this block is executed
+         */
+        orderIndex: number;
+        /**
+         * Whether the tool should be invoked
+         */
+        shouldBeInvoked: boolean;
+        /**
+         * ID of the tool definition
+         */
+        toolDefinitionId: string;
+        /**
+         * Weight of this block in the overall evaluation score (0-100)
+         */
+        weight: number;
+    }
+    interface UnionMember7 {
+        /**
+         * Unique identifier for the block
+         */
+        id: string;
+        /**
+         * Block type identifier
+         */
+        blockType: 'TOXICITY';
+        /**
+         * Optional description of what this block evaluates
+         */
+        description: string | null;
+        /**
+         * Display name of the evaluation block
+         */
+        name: string;
+        /**
+         * Order in which this block is executed
+         */
+        orderIndex: number;
+        /**
+         * Maximum allowed toxicity score (0-1)
+         */
+        threshold: number;
+        /**
+         * Weight of this block in the overall evaluation score (0-100)
+         */
+        weight: number;
+    }
+    interface UnionMember8 {
+        /**
+         * Unique identifier for the block
+         */
+        id: string;
+        /**
+         * Block type identifier
+         */
+        blockType: 'VOCAL_CUE';
+        /**
+         * Optional description of what this block evaluates
+         */
+        description: string | null;
+        /**
+         * Display name of the evaluation block
+         */
+        name: string;
+        /**
+         * Order in which this block is executed
+         */
+        orderIndex: number;
+        /**
+         * The vocal cue to detect (e.g., "pace", "tone", "volume")
+         */
+        selectedCue: string;
+        /**
+         * Minimum confidence threshold for vocal cue detection (0-1)
+         */
+        threshold: number;
+        /**
+         * Weight of this block in the overall evaluation score (0-100)
+         */
+        weight: number;
+    }
+}
+/**
+ * Response containing evaluators and pagination info
+ */
+export interface EvaluationGetEvaluatorsResponse {
+    /**
+     * Array of evaluators with their blocks
+     */
+    data: Array<EvaluationGetEvaluatorsResponse.Data>;
+    /**
+     * Pagination information
+     */
+    pagination: EvaluationGetEvaluatorsResponse.Pagination;
+}
+export declare namespace EvaluationGetEvaluatorsResponse {
+    /**
+     * Evaluator with its configured blocks
+     */
+    interface Data {
+        /**
+         * Unique identifier for the evaluator
+         */
+        id: string;
+        /**
+         * Array of evaluation blocks configured for this evaluator
+         */
+        blocks: Array<Data.UnionMember0 | Data.UnionMember1 | Data.UnionMember2 | Data.UnionMember3 | Data.UnionMember4 | Data.UnionMember5 | Data.UnionMember6 | Data.UnionMember7 | Data.UnionMember8>;
+        /**
+         * ISO timestamp when the evaluator was created
+         */
+        createdAt: string;
+        /**
+         * Optional description of the evaluator
+         */
+        description: string | null;
+        /**
+         * Name of the evaluator
+         */
+        name: string;
+        /**
+         * Unique slug identifier for the evaluator
+         */
+        slug: string;
+        /**
+         * ISO timestamp when the evaluator was last updated
+         */
+        updatedAt: string;
+    }
+    namespace Data {
+        interface UnionMember0 {
+            /**
+             * Unique identifier for the block
+             */
+            id: string;
+            /**
+             * Block type identifier
+             */
+            blockType: 'CUSTOM_PROMPT';
+            /**
+             * Optional description of what this block evaluates
+             */
+            description: string | null;
+            /**
+             * Name of the metric this prompt evaluates
+             */
+            metricName: string;
+            /**
+             * Display name of the evaluation block
+             */
+            name: string;
+            /**
+             * Order in which this block is executed
+             */
+            orderIndex: number;
+            /**
+             * The prompt to evaluate the call against
+             */
+            prompt: string;
+            /**
+             * Minimum score threshold to pass evaluation (0-1)
+             */
+            threshold: number;
+            /**
+             * Weight of this block in the overall evaluation score (0-100)
+             */
+            weight: number;
+        }
+        interface UnionMember1 {
+            /**
+             * Unique identifier for the block
+             */
+            id: string;
+            /**
+             * Block type identifier
+             */
+            blockType: 'DATAFIELD_CHECK';
+            /**
+             * Optional description of what this block evaluates
+             */
+            description: string | null;
+            /**
+             * Criteria for evaluating the property
+             */
+            evaluationCriteria: string;
+            /**
+             * Whether this property must be present
+             */
+            isRequired: boolean;
+            /**
+             * Display name of the evaluation block
+             */
+            name: string;
+            /**
+             * Order in which this block is executed
+             */
+            orderIndex: number;
+            /**
+             * Name of the property to check
+             */
+            propertyName: string;
+            /**
+             * Minimum score threshold to pass evaluation (0-1)
+             */
+            threshold: number;
+            /**
+             * Expected type of the property value
+             */
+            valueType: string;
+            /**
+             * Weight of this block in the overall evaluation score (0-100)
+             */
+            weight: number;
+        }
+        interface UnionMember2 {
+            /**
+             * Unique identifier for the block
+             */
+            id: string;
+            /**
+             * Block type identifier
+             */
+            blockType: 'EMOTION';
+            /**
+             * Optional description of what this block evaluates
+             */
+            description: string | null;
+            /**
+             * Display name of the evaluation block
+             */
+            name: string;
+            /**
+             * Order in which this block is executed
+             */
+            orderIndex: number;
+            /**
+             * The emotion to detect (e.g., "joy", "anger", "sadness")
+             */
+            selectedEmotion: string;
+            /**
+             * Minimum confidence threshold for emotion detection (0-1)
+             */
+            threshold: number;
+            /**
+             * Weight of this block in the overall evaluation score (0-100)
+             */
+            weight: number;
+        }
+        interface UnionMember3 {
+            /**
+             * Unique identifier for the block
+             */
+            id: string;
+            /**
+             * Block type identifier
+             */
+            blockType: 'LATENCY';
+            /**
+             * Optional description of what this block evaluates
+             */
+            description: string | null;
+            /**
+             * Maximum number of silence periods allowed
+             */
+            maxAllowedSilences: number;
+            /**
+             * Minimum duration of silence in milliseconds to be considered
+             */
+            minSilenceDuration: number;
+            /**
+             * Display name of the evaluation block
+             */
+            name: string;
+            /**
+             * Order in which this block is executed
+             */
+            orderIndex: number;
+            /**
+             * Maximum allowed latency score
+             */
+            threshold: number;
+            /**
+             * Weight of this block in the overall evaluation score (0-100)
+             */
+            weight: number;
+        }
+        interface UnionMember4 {
+            /**
+             * Unique identifier for the block
+             */
+            id: string;
+            /**
+             * Block type identifier
+             */
+            blockType: 'POLITENESS';
+            /**
+             * Optional description of what this block evaluates
+             */
+            description: string | null;
+            /**
+             * Display name of the evaluation block
+             */
+            name: string;
+            /**
+             * Order in which this block is executed
+             */
+            orderIndex: number;
+            /**
+             * Minimum politeness score threshold (0-1)
+             */
+            threshold: number;
+            /**
+             * Weight of this block in the overall evaluation score (0-100)
+             */
+            weight: number;
+        }
+        interface UnionMember5 {
+            /**
+             * Unique identifier for the block
+             */
+            id: string;
+            /**
+             * Block type identifier
+             */
+            blockType: 'SENTIMENT';
+            /**
+             * Optional description of what this block evaluates
+             */
+            description: string | null;
+            /**
+             * Display name of the evaluation block
+             */
+            name: string;
+            /**
+             * Order in which this block is executed
+             */
+            orderIndex: number;
+            /**
+             * Minimum sentiment score threshold (0-1)
+             */
+            threshold: number;
+            /**
+             * Weight of this block in the overall evaluation score (0-100)
+             */
+            weight: number;
+        }
+        interface UnionMember6 {
+            /**
+             * Unique identifier for the block
+             */
+            id: string;
+            /**
+             * Block type identifier
+             */
+            blockType: 'TOOL_CALLS';
+            /**
+             * Optional description of what this block evaluates
+             */
+            description: string | null;
+            /**
+             * Condition that must be met for tool invocation
+             */
+            invocationCondition: string | null;
+            /**
+             * Minimum number of times the tool should be invoked
+             */
+            minInvocationCount: number | null;
+            /**
+             * Display name of the evaluation block
+             */
+            name: string;
+            /**
+             * Order in which this block is executed
+             */
+            orderIndex: number;
+            /**
+             * Whether the tool should be invoked
+             */
+            shouldBeInvoked: boolean;
+            /**
+             * ID of the tool definition
+             */
+            toolDefinitionId: string;
+            /**
+             * Weight of this block in the overall evaluation score (0-100)
+             */
+            weight: number;
+        }
+        interface UnionMember7 {
+            /**
+             * Unique identifier for the block
+             */
+            id: string;
+            /**
+             * Block type identifier
+             */
+            blockType: 'TOXICITY';
+            /**
+             * Optional description of what this block evaluates
+             */
+            description: string | null;
+            /**
+             * Display name of the evaluation block
+             */
+            name: string;
+            /**
+             * Order in which this block is executed
+             */
+            orderIndex: number;
+            /**
+             * Maximum allowed toxicity score (0-1)
+             */
+            threshold: number;
+            /**
+             * Weight of this block in the overall evaluation score (0-100)
+             */
+            weight: number;
+        }
+        interface UnionMember8 {
+            /**
+             * Unique identifier for the block
+             */
+            id: string;
+            /**
+             * Block type identifier
+             */
+            blockType: 'VOCAL_CUE';
+            /**
+             * Optional description of what this block evaluates
+             */
+            description: string | null;
+            /**
+             * Display name of the evaluation block
+             */
+            name: string;
+            /**
+             * Order in which this block is executed
+             */
+            orderIndex: number;
+            /**
+             * The vocal cue to detect (e.g., "pace", "tone", "volume")
+             */
+            selectedCue: string;
+            /**
+             * Minimum confidence threshold for vocal cue detection (0-1)
+             */
+            threshold: number;
+            /**
+             * Weight of this block in the overall evaluation score (0-100)
+             */
+            weight: number;
+        }
+    }
+    /**
+     * Pagination information
+     */
+    interface Pagination {
+        /**
+         * Whether there are more evaluators to fetch
+         */
+        hasMore: boolean;
+        /**
+         * Cursor for the next page, null if no more pages
+         */
+        nextCursor: string | null;
+        /**
+         * Total number of evaluators
+         */
+        total: number;
+    }
+}
 export interface EvaluationGetJobResponse {
     /**
      * Evaluation job response payload
@@ -126,107 +870,138 @@ export declare namespace EvaluationGetJobRunsResponse {
     namespace Data {
         interface Data {
             /**
-             * ID of the evaluator run
+             * All block runs for this evaluator, including skipped ones
              */
-            id: string;
+            blockRuns: Array<Data.BlockRun>;
+            evaluator: Data.Evaluator;
+            evidence: Array<Data.Evidence>;
+            metrics: Array<Data.Metric>;
             /**
-             * When the evaluator run completed
-             */
-            completedAt: string | null;
-            /**
-             * Evaluator of the evaluator run
+             * Status of the evaluator run
              */
-            evaluator: Data.Evaluator | null;
+            status: 'PENDING' | 'IN_PROGRESS' | 'COMPLETED' | 'FAILED';
             /**
-             * Evidence of the evaluator run
+             * ID of the evaluator run
              */
-            evidence: Array<Data.Evidence> | null;
+            id?: string;
             /**
-             * Metrics of the evaluator run
+             * When the evaluator run completed
              */
-            metrics: Array<Data.Metric> | null;
+            completedAt?: string | null;
             /**
-             * Score of the evaluator run
+             * Score of the evaluation run (0-1)
              */
-            score: number | null;
+            score?: number | null;
             /**
-             * Score classification of the evaluator run
+             * Score classification of the evaluator run based on score threshold (IRRELEVANT
+             * is mapped to SKIPPED)
              */
-            scoreClassification: 'SUCCESS' | 'FAILURE' | 'IRRELEVANT' | null;
+            scoreClassification?: 'SUCCESS' | 'FAILURE' | 'SKIPPED' | null;
             /**
              * When the evaluator run started
              */
-            startedAt: string | null;
+            startedAt?: string | null;
             /**
-             * Status of the evaluator run
+             * Summary of the evaluation run
              */
-            status: 'PENDING' | 'IN_PROGRESS' | 'COMPLETED' | 'FAILED';
-            /**
-             * Summary of the evaluator run
-             */
-            summary: string | null;
+            summary?: string | null;
         }
         namespace Data {
-            /**
-             * Evaluator of the evaluator run
-             */
+            interface BlockRun {
+                /**
+                 * ID of the block definition
+                 */
+                blockDefinitionId: string;
+                /**
+                 * Name of the evaluation block
+                 */
+                blockName: string;
+                /**
+                 * ID of the block run instance
+                 */
+                blockRunId: string;
+                /**
+                 * When the block run was created
+                 */
+                createdAt: string;
+                /**
+                 * Reason for the outcome (pass/fail explanation or skip reason)
+                 */
+                reason: string | null;
+                /**
+                 * Result of the block run
+                 */
+                result: 'PASSED' | 'FAILED' | 'SKIPPED' | null;
+                /**
+                 * Score of the block run (0-1)
+                 */
+                score: number | null;
+                /**
+                 * Status of the block run
+                 */
+                status: 'PENDING' | 'RUNNING' | 'COMPLETED' | 'FAILED';
+            }
             interface Evaluator {
                 /**
                  * ID of the evaluator
                  */
-                id: string | null;
+                id: string;
                 /**
                  * Name of the evaluator
                  */
-                name: string | null;
+                name: string;
+                /**
+                 * Weight of the evaluator
+                 */
+                weight?: number;
             }
             interface Evidence {
                 /**
-                 * ID of the evidence
+                 * Comment text of the evidence
                  */
-                id: string | null;
+                commentText: string | null;
                 /**
-                 * Comment on the evidence
+                 * Created at of the evidence
                  */
-                commentText: string | null;
+                createdAt: string;
                 /**
-                 * Whether this is a positive example of the metric
+                 * Is positive of the evidence
                  */
-                isPositive: boolean | null;
+                isPositive: boolean;
                 /**
-                 * Snippet of the evidence
+                 * Snippet text of the evidence
                  */
-                snippetText: string | null;
+                snippetText: string;
             }
             interface Metric {
-                /**
-                 * ID of the metric
-                 */
-                id: string | null;
                 /**
                  * Boolean value of the metric
                  */
                 booleanValue: boolean | null;
                 /**
-                 * Confidence of the metric
+                 * Confidence level of the metric (0-1)
                  */
                 confidence: number | null;
+                /**
+                 * Created at of the metric
+                 */
+                createdAt: string;
                 /**
                  * Name of the metric
                  */
-                name: string | null;
+                name: string;
                 /**
                  * Numeric value of the metric
                  */
                 numericValue: number | null;
                 /**
-                 * Reasoning for the metric
+                 * Reasoning of the metric
                  */
                 reasoning: string | null;
                 /**
                  * Role of the metric
                  */
-                role: 'PRIMARY' | 'SECONDARY' | null;
+                role: string;
                 /**
                  * Text value of the metric
                  */
@@ -234,7 +1009,7 @@ export declare namespace EvaluationGetJobRunsResponse {
                 /**
                  * Value type of the metric
                  */
-                valueType: 'NUMERIC' | 'BOOLEAN' | 'TEXT' | null;
+                valueType: string;
             }
         }
         /**
@@ -498,7 +1273,13 @@ export declare namespace EvaluationCreateJobParams {
     }
 }
 export interface EvaluationGetEvaluatorsParams {
+    /**
+     * Cursor for pagination - evaluator ID to start after
+     */
     after?: string;
+    /**
+     * Maximum number of evaluators to return (default: 20, max: 50)
+     */
     limit?: string;
 }
 export interface EvaluationGetJobRunsParams {