@roarkanalytics/sdk 0.331.2 → 0.333.0

package/src/resources/evaluation.ts

@@ -15,6 +15,35 @@ export class Evaluation extends APIResource {
     return this._client.post('/v1/evaluation/job', { body, ...options });
   }
 
+  /**
+   * Returns a specific evaluator with its blocks and configuration.
+   */
+  getEvaluatorById(
+    evaluatorId: string,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<EvaluationGetEvaluatorByIDResponse> {
+    return this._client.get(`/v1/evaluation/evaluators/${evaluatorId}`, options);
+  }
+
+  /**
+   * Returns a list of evaluators with their blocks and configuration for the
+   * authenticated project.
+   */
+  getEvaluators(
+    query?: EvaluationGetEvaluatorsParams,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<EvaluationGetEvaluatorsResponse>;
+  getEvaluators(options?: Core.RequestOptions): Core.APIPromise<EvaluationGetEvaluatorsResponse>;
+  getEvaluators(
+    query: EvaluationGetEvaluatorsParams | Core.RequestOptions = {},
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<EvaluationGetEvaluatorsResponse> {
+    if (isRequestOptions(query)) {
+      return this.getEvaluators({}, query);
+    }
+    return this._client.get('/v1/evaluation/evaluators', { query, ...options });
+  }
+
   /**
    * Retrieve details of a specific evaluation job
    */
@@ -22,42 +51,925 @@ export class Evaluation extends APIResource {
     return this._client.get(`/v1/evaluation/job/${jobId}`, options);
   }
 
-  /**
-   * Retrieve paginated details of a specific evaluation job runs
-   */
-  getJobRuns(
-    jobId: string,
-    query?: EvaluationGetJobRunsParams,
-    options?: Core.RequestOptions,
-  ): Core.APIPromise<EvaluationGetJobRunsResponse>;
-  getJobRuns(jobId: string, options?: Core.RequestOptions): Core.APIPromise<EvaluationGetJobRunsResponse>;
-  getJobRuns(
-    jobId: string,
-    query: EvaluationGetJobRunsParams | Core.RequestOptions = {},
-    options?: Core.RequestOptions,
-  ): Core.APIPromise<EvaluationGetJobRunsResponse> {
-    if (isRequestOptions(query)) {
-      return this.getJobRuns(jobId, {}, query);
+  /**
+   * Retrieve paginated details of a specific evaluation job runs
+   */
+  getJobRuns(
+    jobId: string,
+    query?: EvaluationGetJobRunsParams,
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<EvaluationGetJobRunsResponse>;
+  getJobRuns(jobId: string, options?: Core.RequestOptions): Core.APIPromise<EvaluationGetJobRunsResponse>;
+  getJobRuns(
+    jobId: string,
+    query: EvaluationGetJobRunsParams | Core.RequestOptions = {},
+    options?: Core.RequestOptions,
+  ): Core.APIPromise<EvaluationGetJobRunsResponse> {
+    if (isRequestOptions(query)) {
+      return this.getJobRuns(jobId, {}, query);
+    }
+    return this._client.get(`/v1/evaluation/job/${jobId}/runs`, { query, ...options });
+  }
+}
+
+export interface EvaluationCreateJobResponse {
+  data: EvaluationCreateJobResponse.Data;
+}
+
+export namespace EvaluationCreateJobResponse {
+  export interface Data {
+    /**
+     * ID of the evaluation job
+     */
+    jobId: string;
+
+    /**
+     * Status of the evaluation job
+     */
+    status: 'PENDING' | 'PROCESSING' | 'SUCCESS' | 'FAILURE';
+  }
+}
+
+/**
+ * Evaluator with its configured blocks
+ */
+export interface EvaluationGetEvaluatorByIDResponse {
+  /**
+   * Unique identifier for the evaluator
+   */
+  id: string;
+
+  /**
+   * Array of evaluation blocks configured for this evaluator
+   */
+  blocks: Array<
+    | EvaluationGetEvaluatorByIDResponse.UnionMember0
+    | EvaluationGetEvaluatorByIDResponse.UnionMember1
+    | EvaluationGetEvaluatorByIDResponse.UnionMember2
+    | EvaluationGetEvaluatorByIDResponse.UnionMember3
+    | EvaluationGetEvaluatorByIDResponse.UnionMember4
+    | EvaluationGetEvaluatorByIDResponse.UnionMember5
+    | EvaluationGetEvaluatorByIDResponse.UnionMember6
+    | EvaluationGetEvaluatorByIDResponse.UnionMember7
+    | EvaluationGetEvaluatorByIDResponse.UnionMember8
+  >;
+
+  /**
+   * ISO timestamp when the evaluator was created
+   */
+  createdAt: string;
+
+  /**
+   * Optional description of the evaluator
+   */
+  description: string | null;
+
+  /**
+   * Name of the evaluator
+   */
+  name: string;
+
+  /**
+   * Unique slug identifier for the evaluator
+   */
+  slug: string;
+
+  /**
+   * ISO timestamp when the evaluator was last updated
+   */
+  updatedAt: string;
+}
+
+export namespace EvaluationGetEvaluatorByIDResponse {
+  export interface UnionMember0 {
+    /**
+     * Unique identifier for the block
+     */
+    id: string;
+
+    blockType: 'CUSTOM_PROMPT';
+
+    /**
+     * Optional description of what this block evaluates
+     */
+    description: string | null;
+
+    /**
+     * Name of the metric this prompt evaluates
+     */
+    metricName: string;
+
+    /**
+     * Display name of the evaluation block
+     */
+    name: string;
+
+    /**
+     * Order in which this block is executed
+     */
+    orderIndex: number;
+
+    /**
+     * The prompt to evaluate the call against
+     */
+    prompt: string;
+
+    /**
+     * Minimum score threshold to pass evaluation (0-1)
+     */
+    threshold: number;
+
+    /**
+     * Weight of this block in the overall evaluation score (0-100)
+     */
+    weight: number;
+  }
+
+  export interface UnionMember1 {
+    /**
+     * Unique identifier for the block
+     */
+    id: string;
+
+    blockType: 'DATAFIELD_CHECK';
+
+    /**
+     * Optional description of what this block evaluates
+     */
+    description: string | null;
+
+    /**
+     * Criteria for evaluating the property
+     */
+    evaluationCriteria: string;
+
+    /**
+     * Whether this property must be present
+     */
+    isRequired: boolean;
+
+    /**
+     * Display name of the evaluation block
+     */
+    name: string;
+
+    /**
+     * Order in which this block is executed
+     */
+    orderIndex: number;
+
+    /**
+     * Name of the property to check
+     */
+    propertyName: string;
+
+    /**
+     * Minimum score threshold to pass evaluation (0-1)
+     */
+    threshold: number;
+
+    /**
+     * Expected type of the property value
+     */
+    valueType: string;
+
+    /**
+     * Weight of this block in the overall evaluation score (0-100)
+     */
+    weight: number;
+  }
+
+  export interface UnionMember2 {
+    /**
+     * Unique identifier for the block
+     */
+    id: string;
+
+    blockType: 'EMOTION';
+
+    /**
+     * Optional description of what this block evaluates
+     */
+    description: string | null;
+
+    /**
+     * Display name of the evaluation block
+     */
+    name: string;
+
+    /**
+     * Order in which this block is executed
+     */
+    orderIndex: number;
+
+    /**
+     * The emotion to detect (e.g., "joy", "anger", "sadness")
+     */
+    selectedEmotion: string;
+
+    /**
+     * Minimum confidence threshold for emotion detection (0-1)
+     */
+    threshold: number;
+
+    /**
+     * Weight of this block in the overall evaluation score (0-100)
+     */
+    weight: number;
+  }
+
+  export interface UnionMember3 {
+    /**
+     * Unique identifier for the block
+     */
+    id: string;
+
+    blockType: 'LATENCY';
+
+    /**
+     * Optional description of what this block evaluates
+     */
+    description: string | null;
+
+    /**
+     * Maximum number of silence periods allowed
+     */
+    maxAllowedSilences: number;
+
+    /**
+     * Minimum duration of silence in milliseconds to be considered
+     */
+    minSilenceDuration: number;
+
+    /**
+     * Display name of the evaluation block
+     */
+    name: string;
+
+    /**
+     * Order in which this block is executed
+     */
+    orderIndex: number;
+
+    /**
+     * Maximum allowed latency score
+     */
+    threshold: number;
+
+    /**
+     * Weight of this block in the overall evaluation score (0-100)
+     */
+    weight: number;
+  }
+
+  export interface UnionMember4 {
+    /**
+     * Unique identifier for the block
+     */
+    id: string;
+
+    blockType: 'POLITENESS';
+
+    /**
+     * Optional description of what this block evaluates
+     */
+    description: string | null;
+
+    /**
+     * Display name of the evaluation block
+     */
+    name: string;
+
+    /**
+     * Order in which this block is executed
+     */
+    orderIndex: number;
+
+    /**
+     * Minimum politeness score threshold (0-1)
+     */
+    threshold: number;
+
+    /**
+     * Weight of this block in the overall evaluation score (0-100)
+     */
+    weight: number;
+  }
+
+  export interface UnionMember5 {
+    /**
+     * Unique identifier for the block
+     */
+    id: string;
+
+    blockType: 'SENTIMENT';
+
+    /**
+     * Optional description of what this block evaluates
+     */
+    description: string | null;
+
+    /**
+     * Display name of the evaluation block
+     */
+    name: string;
+
+    /**
+     * Order in which this block is executed
+     */
+    orderIndex: number;
+
+    /**
+     * Minimum sentiment score threshold (0-1)
+     */
+    threshold: number;
+
+    /**
+     * Weight of this block in the overall evaluation score (0-100)
+     */
+    weight: number;
+  }
+
+  export interface UnionMember6 {
+    /**
+     * Unique identifier for the block
+     */
+    id: string;
+
+    blockType: 'TOOL_CALLS';
+
+    /**
+     * Optional description of what this block evaluates
+     */
+    description: string | null;
+
+    /**
+     * Condition that must be met for tool invocation
+     */
+    invocationCondition: string | null;
+
+    /**
+     * Minimum number of times the tool should be invoked
+     */
+    minInvocationCount: number | null;
+
+    /**
+     * Display name of the evaluation block
+     */
+    name: string;
+
+    /**
+     * Order in which this block is executed
+     */
+    orderIndex: number;
+
+    /**
+     * Whether the tool should be invoked
+     */
+    shouldBeInvoked: boolean;
+
+    /**
+     * ID of the tool definition
+     */
+    toolDefinitionId: string;
+
+    /**
+     * Weight of this block in the overall evaluation score (0-100)
+     */
+    weight: number;
+  }
+
+  export interface UnionMember7 {
+    /**
+     * Unique identifier for the block
+     */
+    id: string;
+
+    blockType: 'TOXICITY';
+
+    /**
+     * Optional description of what this block evaluates
+     */
+    description: string | null;
+
+    /**
+     * Display name of the evaluation block
+     */
+    name: string;
+
+    /**
+     * Order in which this block is executed
+     */
+    orderIndex: number;
+
+    /**
+     * Maximum allowed toxicity score (0-1)
+     */
+    threshold: number;
+
+    /**
+     * Weight of this block in the overall evaluation score (0-100)
+     */
+    weight: number;
+  }
+
+  export interface UnionMember8 {
+    /**
+     * Unique identifier for the block
+     */
+    id: string;
+
+    blockType: 'VOCAL_CUE';
+
+    /**
+     * Optional description of what this block evaluates
+     */
+    description: string | null;
+
+    /**
+     * Display name of the evaluation block
+     */
+    name: string;
+
+    /**
+     * Order in which this block is executed
+     */
+    orderIndex: number;
+
+    /**
+     * The vocal cue to detect (e.g., "pace", "tone", "volume")
+     */
+    selectedCue: string;
+
+    /**
+     * Minimum confidence threshold for vocal cue detection (0-1)
+     */
+    threshold: number;
+
+    /**
+     * Weight of this block in the overall evaluation score (0-100)
+     */
+    weight: number;
+  }
+}
+
+/**
+ * Response containing evaluators and pagination info
+ */
+export interface EvaluationGetEvaluatorsResponse {
+  /**
+   * Array of evaluators with their blocks
+   */
+  data: Array<EvaluationGetEvaluatorsResponse.Data>;
+
+  /**
+   * Pagination information
+   */
+  pagination: EvaluationGetEvaluatorsResponse.Pagination;
+}
+
+export namespace EvaluationGetEvaluatorsResponse {
+  /**
+   * Evaluator with its configured blocks
+   */
+  export interface Data {
+    /**
+     * Unique identifier for the evaluator
+     */
+    id: string;
+
+    /**
+     * Array of evaluation blocks configured for this evaluator
+     */
+    blocks: Array<
+      | Data.UnionMember0
+      | Data.UnionMember1
+      | Data.UnionMember2
+      | Data.UnionMember3
+      | Data.UnionMember4
+      | Data.UnionMember5
+      | Data.UnionMember6
+      | Data.UnionMember7
+      | Data.UnionMember8
+    >;
+
+    /**
+     * ISO timestamp when the evaluator was created
+     */
+    createdAt: string;
+
+    /**
+     * Optional description of the evaluator
+     */
+    description: string | null;
+
+    /**
+     * Name of the evaluator
+     */
+    name: string;
+
+    /**
+     * Unique slug identifier for the evaluator
+     */
+    slug: string;
+
+    /**
+     * ISO timestamp when the evaluator was last updated
+     */
+    updatedAt: string;
+  }
+
+  export namespace Data {
+    export interface UnionMember0 {
+      /**
+       * Unique identifier for the block
+       */
+      id: string;
+
+      blockType: 'CUSTOM_PROMPT';
+
+      /**
+       * Optional description of what this block evaluates
+       */
+      description: string | null;
+
+      /**
+       * Name of the metric this prompt evaluates
+       */
+      metricName: string;
+
+      /**
+       * Display name of the evaluation block
+       */
+      name: string;
+
+      /**
+       * Order in which this block is executed
+       */
+      orderIndex: number;
+
+      /**
+       * The prompt to evaluate the call against
+       */
+      prompt: string;
+
+      /**
+       * Minimum score threshold to pass evaluation (0-1)
+       */
+      threshold: number;
+
+      /**
+       * Weight of this block in the overall evaluation score (0-100)
+       */
+      weight: number;
+    }
+
+    export interface UnionMember1 {
+      /**
+       * Unique identifier for the block
+       */
+      id: string;
+
+      blockType: 'DATAFIELD_CHECK';
+
+      /**
+       * Optional description of what this block evaluates
+       */
+      description: string | null;
+
+      /**
+       * Criteria for evaluating the property
+       */
+      evaluationCriteria: string;
+
+      /**
+       * Whether this property must be present
+       */
+      isRequired: boolean;
+
+      /**
+       * Display name of the evaluation block
+       */
+      name: string;
+
+      /**
+       * Order in which this block is executed
+       */
+      orderIndex: number;
+
+      /**
+       * Name of the property to check
+       */
+      propertyName: string;
+
+      /**
+       * Minimum score threshold to pass evaluation (0-1)
+       */
+      threshold: number;
+
+      /**
+       * Expected type of the property value
+       */
+      valueType: string;
+
+      /**
+       * Weight of this block in the overall evaluation score (0-100)
+       */
+      weight: number;
+    }
+
+    export interface UnionMember2 {
+      /**
+       * Unique identifier for the block
+       */
+      id: string;
+
+      blockType: 'EMOTION';
+
+      /**
+       * Optional description of what this block evaluates
+       */
+      description: string | null;
+
+      /**
+       * Display name of the evaluation block
+       */
+      name: string;
+
+      /**
+       * Order in which this block is executed
+       */
+      orderIndex: number;
+
+      /**
+       * The emotion to detect (e.g., "joy", "anger", "sadness")
+       */
+      selectedEmotion: string;
+
+      /**
+       * Minimum confidence threshold for emotion detection (0-1)
+       */
+      threshold: number;
+
+      /**
+       * Weight of this block in the overall evaluation score (0-100)
+       */
+      weight: number;
+    }
+
+    export interface UnionMember3 {
+      /**
+       * Unique identifier for the block
+       */
+      id: string;
+
+      blockType: 'LATENCY';
+
+      /**
+       * Optional description of what this block evaluates
+       */
+      description: string | null;
+
+      /**
+       * Maximum number of silence periods allowed
+       */
+      maxAllowedSilences: number;
+
+      /**
+       * Minimum duration of silence in milliseconds to be considered
+       */
+      minSilenceDuration: number;
+
+      /**
+       * Display name of the evaluation block
+       */
+      name: string;
+
+      /**
+       * Order in which this block is executed
+       */
+      orderIndex: number;
+
+      /**
+       * Maximum allowed latency score
+       */
+      threshold: number;
+
+      /**
+       * Weight of this block in the overall evaluation score (0-100)
+       */
+      weight: number;
+    }
+
+    export interface UnionMember4 {
+      /**
+       * Unique identifier for the block
+       */
+      id: string;
+
+      blockType: 'POLITENESS';
+
+      /**
+       * Optional description of what this block evaluates
+       */
+      description: string | null;
+
+      /**
+       * Display name of the evaluation block
+       */
+      name: string;
+
+      /**
+       * Order in which this block is executed
+       */
+      orderIndex: number;
+
+      /**
+       * Minimum politeness score threshold (0-1)
+       */
+      threshold: number;
+
+      /**
+       * Weight of this block in the overall evaluation score (0-100)
+       */
+      weight: number;
+    }
+
+    export interface UnionMember5 {
+      /**
+       * Unique identifier for the block
+       */
+      id: string;
+
+      blockType: 'SENTIMENT';
+
+      /**
+       * Optional description of what this block evaluates
+       */
+      description: string | null;
+
+      /**
+       * Display name of the evaluation block
+       */
+      name: string;
+
+      /**
+       * Order in which this block is executed
+       */
+      orderIndex: number;
+
+      /**
+       * Minimum sentiment score threshold (0-1)
+       */
+      threshold: number;
+
+      /**
+       * Weight of this block in the overall evaluation score (0-100)
+       */
+      weight: number;
+    }
+
+    export interface UnionMember6 {
+      /**
+       * Unique identifier for the block
+       */
+      id: string;
+
+      blockType: 'TOOL_CALLS';
+
+      /**
+       * Optional description of what this block evaluates
+       */
+      description: string | null;
+
+      /**
+       * Condition that must be met for tool invocation
+       */
+      invocationCondition: string | null;
+
+      /**
+       * Minimum number of times the tool should be invoked
+       */
+      minInvocationCount: number | null;
+
+      /**
+       * Display name of the evaluation block
+       */
+      name: string;
+
+      /**
+       * Order in which this block is executed
+       */
+      orderIndex: number;
+
+      /**
+       * Whether the tool should be invoked
+       */
+      shouldBeInvoked: boolean;
+
+      /**
+       * ID of the tool definition
+       */
+      toolDefinitionId: string;
+
+      /**
+       * Weight of this block in the overall evaluation score (0-100)
+       */
+      weight: number;
+    }
+
+    export interface UnionMember7 {
+      /**
+       * Unique identifier for the block
+       */
+      id: string;
+
+      blockType: 'TOXICITY';
+
+      /**
+       * Optional description of what this block evaluates
+       */
+      description: string | null;
+
+      /**
+       * Display name of the evaluation block
+       */
+      name: string;
+
+      /**
+       * Order in which this block is executed
+       */
+      orderIndex: number;
+
+      /**
+       * Maximum allowed toxicity score (0-1)
+       */
+      threshold: number;
+
+      /**
+       * Weight of this block in the overall evaluation score (0-100)
+       */
+      weight: number;
+    }
+
+    export interface UnionMember8 {
+      /**
+       * Unique identifier for the block
+       */
+      id: string;
+
+      blockType: 'VOCAL_CUE';
+
+      /**
+       * Optional description of what this block evaluates
+       */
+      description: string | null;
+
+      /**
+       * Display name of the evaluation block
+       */
+      name: string;
+
+      /**
+       * Order in which this block is executed
+       */
+      orderIndex: number;
+
+      /**
+       * The vocal cue to detect (e.g., "pace", "tone", "volume")
+       */
+      selectedCue: string;
+
+      /**
+       * Minimum confidence threshold for vocal cue detection (0-1)
+       */
+      threshold: number;
+
+      /**
+       * Weight of this block in the overall evaluation score (0-100)
+       */
+      weight: number;
     }
-    return this._client.get(`/v1/evaluation/job/${jobId}/runs`, { query, ...options });
   }
-}
 
-export interface EvaluationCreateJobResponse {
-  data: EvaluationCreateJobResponse.Data;
-}
+  /**
+   * Pagination information
+   */
+  export interface Pagination {
+    /**
+     * Whether there are more evaluators to fetch
+     */
+    hasMore: boolean;
 
-export namespace EvaluationCreateJobResponse {
-  export interface Data {
     /**
-     * ID of the evaluation job
+     * Cursor for the next page, null if no more pages
      */
-    jobId: string;
+    nextCursor: string | null;
 
     /**
-     * Status of the evaluation job
+     * Total number of evaluators
      */
-    status: 'PENDING' | 'PROCESSING' | 'SUCCESS' | 'FAILURE';
+    total: number;
   }
 }
 
@@ -627,6 +1539,18 @@ export namespace EvaluationCreateJobParams {
   }
 }
 
+export interface EvaluationGetEvaluatorsParams {
+  /**
+   * Cursor for pagination - evaluator ID to start after
+   */
+  after?: string;
+
+  /**
+   * Maximum number of evaluators to return (default: 20, max: 50)
+   */
+  limit?: string;
+}
+
 export interface EvaluationGetJobRunsParams {
   /**
    * Number of items to return per page
@@ -642,9 +1566,12 @@ export interface EvaluationGetJobRunsParams {
 export declare namespace Evaluation {
   export {
     type EvaluationCreateJobResponse as EvaluationCreateJobResponse,
+    type EvaluationGetEvaluatorByIDResponse as EvaluationGetEvaluatorByIDResponse,
+    type EvaluationGetEvaluatorsResponse as EvaluationGetEvaluatorsResponse,
     type EvaluationGetJobResponse as EvaluationGetJobResponse,
     type EvaluationGetJobRunsResponse as EvaluationGetJobRunsResponse,
     type EvaluationCreateJobParams as EvaluationCreateJobParams,
+    type EvaluationGetEvaluatorsParams as EvaluationGetEvaluatorsParams,
     type EvaluationGetJobRunsParams as EvaluationGetJobRunsParams,
   };
 }