openai 4.96.2 → 4.98.0

package/src/resources/evals/evals.ts

@@ -4,6 +4,7 @@ import { APIResource } from '../../resource';
 import { isRequestOptions } from '../../core';
 import * as Core from '../../core';
 import * as Shared from '../shared';
+import * as GraderModelsAPI from '../graders/grader-models';
 import * as ResponsesAPI from '../responses/responses';
 import * as RunsAPI from './runs/runs';
 import {
@@ -103,83 +104,6 @@ export interface EvalCustomDataSourceConfig {
   type: 'custom';
 }
 
-/**
- * A LabelModelGrader object which uses a model to assign labels to each item in
- * the evaluation.
- */
-export interface EvalLabelModelGrader {
-  input: Array<EvalLabelModelGrader.Input>;
-
-  /**
-   * The labels to assign to each item in the evaluation.
-   */
-  labels: Array<string>;
-
-  /**
-   * The model to use for the evaluation. Must support structured outputs.
-   */
-  model: string;
-
-  /**
-   * The name of the grader.
-   */
-  name: string;
-
-  /**
-   * The labels that indicate a passing result. Must be a subset of labels.
-   */
-  passing_labels: Array<string>;
-
-  /**
-   * The object type, which is always `label_model`.
-   */
-  type: 'label_model';
-}
-
-export namespace EvalLabelModelGrader {
-  /**
-   * A message input to the model with a role indicating instruction following
-   * hierarchy. Instructions given with the `developer` or `system` role take
-   * precedence over instructions given with the `user` role. Messages with the
-   * `assistant` role are presumed to have been generated by the model in previous
-   * interactions.
-   */
-  export interface Input {
-    /**
-     * Text inputs to the model - can contain template strings.
-     */
-    content: string | ResponsesAPI.ResponseInputText | Input.OutputText;
-
-    /**
-     * The role of the message input. One of `user`, `assistant`, `system`, or
-     * `developer`.
-     */
-    role: 'user' | 'assistant' | 'system' | 'developer';
-
-    /**
-     * The type of the message input. Always `message`.
-     */
-    type?: 'message';
-  }
-
-  export namespace Input {
-    /**
-     * A text output from the model.
-     */
-    export interface OutputText {
-      /**
-       * The text output from the model.
-       */
-      text: string;
-
-      /**
-       * The type of the output text. Always `output_text`.
-       */
-      type: 'output_text';
-    }
-  }
-}
-
 /**
  * A StoredCompletionsDataSourceConfig which specifies the metadata property of
  * your stored completions query. This is usually metadata like `usecase=chatbot`
@@ -210,83 +134,6 @@ export interface EvalStoredCompletionsDataSourceConfig {
   metadata?: Shared.Metadata | null;
 }
 
-/**
- * A StringCheckGrader object that performs a string comparison between input and
- * reference using a specified operation.
- */
-export interface EvalStringCheckGrader {
-  /**
-   * The input text. This may include template strings.
-   */
-  input: string;
-
-  /**
-   * The name of the grader.
-   */
-  name: string;
-
-  /**
-   * The string check operation to perform. One of `eq`, `ne`, `like`, or `ilike`.
-   */
-  operation: 'eq' | 'ne' | 'like' | 'ilike';
-
-  /**
-   * The reference text. This may include template strings.
-   */
-  reference: string;
-
-  /**
-   * The object type, which is always `string_check`.
-   */
-  type: 'string_check';
-}
-
-/**
- * A TextSimilarityGrader object which grades text based on similarity metrics.
- */
-export interface EvalTextSimilarityGrader {
-  /**
-   * The evaluation metric to use. One of `fuzzy_match`, `bleu`, `gleu`, `meteor`,
-   * `rouge_1`, `rouge_2`, `rouge_3`, `rouge_4`, `rouge_5`, or `rouge_l`.
-   */
-  evaluation_metric:
-    | 'fuzzy_match'
-    | 'bleu'
-    | 'gleu'
-    | 'meteor'
-    | 'rouge_1'
-    | 'rouge_2'
-    | 'rouge_3'
-    | 'rouge_4'
-    | 'rouge_5'
-    | 'rouge_l';
-
-  /**
-   * The text being graded.
-   */
-  input: string;
-
-  /**
-   * A float score where a value greater than or equal indicates a passing grade.
-   */
-  pass_threshold: number;
-
-  /**
-   * The text being graded against.
-   */
-  reference: string;
-
-  /**
-   * The type of grader.
-   */
-  type: 'text_similarity';
-
-  /**
-   * The name of the grader.
-   */
-  name?: string;
-}
-
 /**
  * An Eval object with a data source config and testing criteria. An Eval
  * represents a task to be done for your LLM integration. Like:
@@ -335,39 +182,29 @@ export interface EvalCreateResponse {
    * A list of testing criteria.
    */
   testing_criteria: Array<
-    | EvalLabelModelGrader
-    | EvalStringCheckGrader
-    | EvalTextSimilarityGrader
-    | EvalCreateResponse.Python
-    | EvalCreateResponse.ScoreModel
+    | GraderModelsAPI.LabelModelGrader
+    | GraderModelsAPI.StringCheckGrader
+    | EvalCreateResponse.EvalGraderTextSimilarity
+    | EvalCreateResponse.EvalGraderPython
+    | EvalCreateResponse.EvalGraderScoreModel
   >;
 }
 
 export namespace EvalCreateResponse {
   /**
-   * A PythonGrader object that runs a python script on the input.
+   * A TextSimilarityGrader object which grades text based on similarity metrics.
    */
-  export interface Python {
-    /**
-     * The name of the grader.
-     */
-    name: string;
-
+  export interface EvalGraderTextSimilarity extends GraderModelsAPI.TextSimilarityGrader {
     /**
-     * The source code of the python script.
-     */
-    source: string;
-
-    /**
-     * The object type, which is always `python`.
-     */
-    type: 'python';
-
-    /**
-     * The image tag to use for the python script.
+     * The threshold for the score.
      */
-    image_tag?: string;
+    pass_threshold: number;
+  }
 
+  /**
+   * A PythonGrader object that runs a python script on the input.
+   */
+  export interface EvalGraderPython extends GraderModelsAPI.PythonGrader {
     /**
      * The threshold for the score.
      */
@@ -377,85 +214,11 @@ export namespace EvalCreateResponse {
   /**
    * A ScoreModelGrader object that uses a model to assign a score to the input.
    */
-  export interface ScoreModel {
-    /**
-     * The input text. This may include template strings.
-     */
-    input: Array<ScoreModel.Input>;
-
-    /**
-     * The model to use for the evaluation.
-     */
-    model: string;
-
-    /**
-     * The name of the grader.
-     */
-    name: string;
-
-    /**
-     * The object type, which is always `score_model`.
-     */
-    type: 'score_model';
-
+  export interface EvalGraderScoreModel extends GraderModelsAPI.ScoreModelGrader {
     /**
      * The threshold for the score.
      */
     pass_threshold?: number;
-
-    /**
-     * The range of the score. Defaults to `[0, 1]`.
-     */
-    range?: Array<number>;
-
-    /**
-     * The sampling parameters for the model.
-     */
-    sampling_params?: unknown;
-  }
-
-  export namespace ScoreModel {
-    /**
-     * A message input to the model with a role indicating instruction following
-     * hierarchy. Instructions given with the `developer` or `system` role take
-     * precedence over instructions given with the `user` role. Messages with the
-     * `assistant` role are presumed to have been generated by the model in previous
-     * interactions.
-     */
-    export interface Input {
-      /**
-       * Text inputs to the model - can contain template strings.
-       */
-      content: string | ResponsesAPI.ResponseInputText | Input.OutputText;
-
-      /**
-       * The role of the message input. One of `user`, `assistant`, `system`, or
-       * `developer`.
-       */
-      role: 'user' | 'assistant' | 'system' | 'developer';
-
-      /**
-       * The type of the message input. Always `message`.
-       */
-      type?: 'message';
-    }
-
-    export namespace Input {
-      /**
-       * A text output from the model.
-       */
-      export interface OutputText {
-        /**
-         * The text output from the model.
-         */
-        text: string;
-
-        /**
-         * The type of the output text. Always `output_text`.
-         */
-        type: 'output_text';
-      }
-    }
   }
 }
 
@@ -507,39 +270,29 @@ export interface EvalRetrieveResponse {
    * A list of testing criteria.
    */
   testing_criteria: Array<
-    | EvalLabelModelGrader
-    | EvalStringCheckGrader
-    | EvalTextSimilarityGrader
-    | EvalRetrieveResponse.Python
-    | EvalRetrieveResponse.ScoreModel
+    | GraderModelsAPI.LabelModelGrader
+    | GraderModelsAPI.StringCheckGrader
+    | EvalRetrieveResponse.EvalGraderTextSimilarity
+    | EvalRetrieveResponse.EvalGraderPython
+    | EvalRetrieveResponse.EvalGraderScoreModel
   >;
 }
 
 export namespace EvalRetrieveResponse {
   /**
-   * A PythonGrader object that runs a python script on the input.
+   * A TextSimilarityGrader object which grades text based on similarity metrics.
    */
-  export interface Python {
+  export interface EvalGraderTextSimilarity extends GraderModelsAPI.TextSimilarityGrader {
     /**
-     * The name of the grader.
-     */
-    name: string;
-
-    /**
-     * The source code of the python script.
-     */
-    source: string;
-
-    /**
-     * The object type, which is always `python`.
-     */
-    type: 'python';
-
-    /**
-     * The image tag to use for the python script.
+     * The threshold for the score.
      */
-    image_tag?: string;
+    pass_threshold: number;
+  }
 
+  /**
+   * A PythonGrader object that runs a python script on the input.
+   */
+  export interface EvalGraderPython extends GraderModelsAPI.PythonGrader {
     /**
      * The threshold for the score.
      */
@@ -549,85 +302,11 @@ export namespace EvalRetrieveResponse {
   /**
    * A ScoreModelGrader object that uses a model to assign a score to the input.
    */
-  export interface ScoreModel {
-    /**
-     * The input text. This may include template strings.
-     */
-    input: Array<ScoreModel.Input>;
-
-    /**
-     * The model to use for the evaluation.
-     */
-    model: string;
-
-    /**
-     * The name of the grader.
-     */
-    name: string;
-
-    /**
-     * The object type, which is always `score_model`.
-     */
-    type: 'score_model';
-
+  export interface EvalGraderScoreModel extends GraderModelsAPI.ScoreModelGrader {
     /**
      * The threshold for the score.
      */
     pass_threshold?: number;
-
-    /**
-     * The range of the score. Defaults to `[0, 1]`.
-     */
-    range?: Array<number>;
-
-    /**
-     * The sampling parameters for the model.
-     */
-    sampling_params?: unknown;
-  }
-
-  export namespace ScoreModel {
-    /**
-     * A message input to the model with a role indicating instruction following
-     * hierarchy. Instructions given with the `developer` or `system` role take
-     * precedence over instructions given with the `user` role. Messages with the
-     * `assistant` role are presumed to have been generated by the model in previous
-     * interactions.
-     */
-    export interface Input {
-      /**
-       * Text inputs to the model - can contain template strings.
-       */
-      content: string | ResponsesAPI.ResponseInputText | Input.OutputText;
-
-      /**
-       * The role of the message input. One of `user`, `assistant`, `system`, or
-       * `developer`.
-       */
-      role: 'user' | 'assistant' | 'system' | 'developer';
-
-      /**
-       * The type of the message input. Always `message`.
-       */
-      type?: 'message';
-    }
-
-    export namespace Input {
-      /**
-       * A text output from the model.
-       */
-      export interface OutputText {
-        /**
-         * The text output from the model.
-         */
-        text: string;
-
-        /**
-         * The type of the output text. Always `output_text`.
-         */
-        type: 'output_text';
-      }
-    }
   }
 }
 
@@ -679,39 +358,29 @@ export interface EvalUpdateResponse {
    * A list of testing criteria.
    */
   testing_criteria: Array<
-    | EvalLabelModelGrader
-    | EvalStringCheckGrader
-    | EvalTextSimilarityGrader
-    | EvalUpdateResponse.Python
-    | EvalUpdateResponse.ScoreModel
+    | GraderModelsAPI.LabelModelGrader
+    | GraderModelsAPI.StringCheckGrader
+    | EvalUpdateResponse.EvalGraderTextSimilarity
+    | EvalUpdateResponse.EvalGraderPython
+    | EvalUpdateResponse.EvalGraderScoreModel
   >;
 }
 
 export namespace EvalUpdateResponse {
   /**
-   * A PythonGrader object that runs a python script on the input.
+   * A TextSimilarityGrader object which grades text based on similarity metrics.
    */
-  export interface Python {
-    /**
-     * The name of the grader.
-     */
-    name: string;
-
-    /**
-     * The source code of the python script.
-     */
-    source: string;
-
-    /**
-     * The object type, which is always `python`.
-     */
-    type: 'python';
-
+  export interface EvalGraderTextSimilarity extends GraderModelsAPI.TextSimilarityGrader {
     /**
-     * The image tag to use for the python script.
+     * The threshold for the score.
      */
-    image_tag?: string;
+    pass_threshold: number;
+  }
 
+  /**
+   * A PythonGrader object that runs a python script on the input.
+   */
+  export interface EvalGraderPython extends GraderModelsAPI.PythonGrader {
     /**
      * The threshold for the score.
      */
@@ -721,85 +390,11 @@ export namespace EvalUpdateResponse {
   /**
    * A ScoreModelGrader object that uses a model to assign a score to the input.
    */
-  export interface ScoreModel {
-    /**
-     * The input text. This may include template strings.
-     */
-    input: Array<ScoreModel.Input>;
-
-    /**
-     * The model to use for the evaluation.
-     */
-    model: string;
-
-    /**
-     * The name of the grader.
-     */
-    name: string;
-
-    /**
-     * The object type, which is always `score_model`.
-     */
-    type: 'score_model';
-
+  export interface EvalGraderScoreModel extends GraderModelsAPI.ScoreModelGrader {
     /**
      * The threshold for the score.
      */
     pass_threshold?: number;
-
-    /**
-     * The range of the score. Defaults to `[0, 1]`.
-     */
-    range?: Array<number>;
-
-    /**
-     * The sampling parameters for the model.
-     */
-    sampling_params?: unknown;
-  }
-
-  export namespace ScoreModel {
-    /**
-     * A message input to the model with a role indicating instruction following
-     * hierarchy. Instructions given with the `developer` or `system` role take
-     * precedence over instructions given with the `user` role. Messages with the
-     * `assistant` role are presumed to have been generated by the model in previous
-     * interactions.
-     */
-    export interface Input {
-      /**
-       * Text inputs to the model - can contain template strings.
-       */
-      content: string | ResponsesAPI.ResponseInputText | Input.OutputText;
-
-      /**
-       * The role of the message input. One of `user`, `assistant`, `system`, or
-       * `developer`.
-       */
-      role: 'user' | 'assistant' | 'system' | 'developer';
-
-      /**
-       * The type of the message input. Always `message`.
-       */
-      type?: 'message';
-    }
-
-    export namespace Input {
-      /**
-       * A text output from the model.
-       */
-      export interface OutputText {
-        /**
-         * The text output from the model.
-         */
-        text: string;
-
-        /**
-         * The type of the output text. Always `output_text`.
-         */
-        type: 'output_text';
-      }
-    }
   }
 }
 
@@ -851,39 +446,29 @@ export interface EvalListResponse {
    * A list of testing criteria.
    */
   testing_criteria: Array<
-    | EvalLabelModelGrader
-    | EvalStringCheckGrader
-    | EvalTextSimilarityGrader
-    | EvalListResponse.Python
-    | EvalListResponse.ScoreModel
+    | GraderModelsAPI.LabelModelGrader
+    | GraderModelsAPI.StringCheckGrader
+    | EvalListResponse.EvalGraderTextSimilarity
+    | EvalListResponse.EvalGraderPython
+    | EvalListResponse.EvalGraderScoreModel
   >;
 }
 
 export namespace EvalListResponse {
   /**
-   * A PythonGrader object that runs a python script on the input.
+   * A TextSimilarityGrader object which grades text based on similarity metrics.
    */
-  export interface Python {
-    /**
-     * The name of the grader.
-     */
-    name: string;
-
-    /**
-     * The source code of the python script.
-     */
-    source: string;
-
+  export interface EvalGraderTextSimilarity extends GraderModelsAPI.TextSimilarityGrader {
     /**
-     * The object type, which is always `python`.
-     */
-    type: 'python';
-
-    /**
-     * The image tag to use for the python script.
+     * The threshold for the score.
      */
-    image_tag?: string;
+    pass_threshold: number;
+  }
 
+  /**
+   * A PythonGrader object that runs a python script on the input.
+   */
+  export interface EvalGraderPython extends GraderModelsAPI.PythonGrader {
     /**
      * The threshold for the score.
      */
@@ -893,85 +478,11 @@ export namespace EvalListResponse {
   /**
    * A ScoreModelGrader object that uses a model to assign a score to the input.
    */
-  export interface ScoreModel {
-    /**
-     * The input text. This may include template strings.
-     */
-    input: Array<ScoreModel.Input>;
-
-    /**
-     * The model to use for the evaluation.
-     */
-    model: string;
-
-    /**
-     * The name of the grader.
-     */
-    name: string;
-
-    /**
-     * The object type, which is always `score_model`.
-     */
-    type: 'score_model';
-
+  export interface EvalGraderScoreModel extends GraderModelsAPI.ScoreModelGrader {
     /**
      * The threshold for the score.
      */
     pass_threshold?: number;
-
-    /**
-     * The range of the score. Defaults to `[0, 1]`.
-     */
-    range?: Array<number>;
-
-    /**
-     * The sampling parameters for the model.
-     */
-    sampling_params?: unknown;
-  }
-
-  export namespace ScoreModel {
-    /**
-     * A message input to the model with a role indicating instruction following
-     * hierarchy. Instructions given with the `developer` or `system` role take
-     * precedence over instructions given with the `user` role. Messages with the
-     * `assistant` role are presumed to have been generated by the model in previous
-     * interactions.
-     */
-    export interface Input {
-      /**
-       * Text inputs to the model - can contain template strings.
-       */
-      content: string | ResponsesAPI.ResponseInputText | Input.OutputText;
-
-      /**
-       * The role of the message input. One of `user`, `assistant`, `system`, or
-       * `developer`.
-       */
-      role: 'user' | 'assistant' | 'system' | 'developer';
-
-      /**
-       * The type of the message input. Always `message`.
-       */
-      type?: 'message';
-    }
-
-    export namespace Input {
-      /**
-       * A text output from the model.
-       */
-      export interface OutputText {
-        /**
-         * The text output from the model.
-         */
-        text: string;
-
-        /**
-         * The type of the output text. Always `output_text`.
-         */
-        type: 'output_text';
-      }
-    }
   }
 }
 
@@ -987,15 +498,15 @@ export interface EvalCreateParams {
   /**
    * The configuration for the data source used for the evaluation runs.
    */
-  data_source_config: EvalCreateParams.Custom | EvalCreateParams.Logs;
+  data_source_config: EvalCreateParams.Custom | EvalCreateParams.StoredCompletions;
 
   /**
    * A list of graders for all eval runs in this group.
    */
   testing_criteria: Array<
     | EvalCreateParams.LabelModel
-    | EvalStringCheckGrader
-    | EvalTextSimilarityGrader
+    | GraderModelsAPI.StringCheckGrader
+    | EvalCreateParams.TextSimilarity
     | EvalCreateParams.Python
     | EvalCreateParams.ScoreModel
   >;
@@ -1048,14 +559,14 @@ export namespace EvalCreateParams {
    * completions query. This is usually metadata like `usecase=chatbot` or
    * `prompt-version=v2`, etc.
    */
-  export interface Logs {
+  export interface StoredCompletions {
     /**
-     * The type of data source. Always `logs`.
+     * The type of data source. Always `stored_completions`.
      */
-    type: 'logs';
+    type: 'stored_completions';
 
     /**
-     * Metadata filters for the logs data source.
+     * Metadata filters for the stored completions data source.
      */
     metadata?: Record<string, unknown>;
   }
@@ -1154,29 +665,19 @@ export namespace EvalCreateParams {
   }
 
   /**
-   * A PythonGrader object that runs a python script on the input.
+   * A TextSimilarityGrader object which grades text based on similarity metrics.
    */
-  export interface Python {
+  export interface TextSimilarity extends GraderModelsAPI.TextSimilarityGrader {
     /**
-     * The name of the grader.
-     */
-    name: string;
-
-    /**
-     * The source code of the python script.
-     */
-    source: string;
-
-    /**
-     * The object type, which is always `python`.
-     */
-    type: 'python';
-
-    /**
-     * The image tag to use for the python script.
+     * The threshold for the score.
      */
-    image_tag?: string;
+    pass_threshold: number;
+  }
 
+  /**
+   * A PythonGrader object that runs a python script on the input.
+   */
+  export interface Python extends GraderModelsAPI.PythonGrader {
     /**
      * The threshold for the score.
      */
@@ -1186,85 +687,11 @@ export namespace EvalCreateParams {
   /**
    * A ScoreModelGrader object that uses a model to assign a score to the input.
    */
-  export interface ScoreModel {
-    /**
-     * The input text. This may include template strings.
-     */
-    input: Array<ScoreModel.Input>;
-
-    /**
-     * The model to use for the evaluation.
-     */
-    model: string;
-
-    /**
-     * The name of the grader.
-     */
-    name: string;
-
-    /**
-     * The object type, which is always `score_model`.
-     */
-    type: 'score_model';
-
+  export interface ScoreModel extends GraderModelsAPI.ScoreModelGrader {
     /**
      * The threshold for the score.
      */
     pass_threshold?: number;
-
-    /**
-     * The range of the score. Defaults to `[0, 1]`.
-     */
-    range?: Array<number>;
-
-    /**
-     * The sampling parameters for the model.
-     */
-    sampling_params?: unknown;
-  }
-
-  export namespace ScoreModel {
-    /**
-     * A message input to the model with a role indicating instruction following
-     * hierarchy. Instructions given with the `developer` or `system` role take
-     * precedence over instructions given with the `user` role. Messages with the
-     * `assistant` role are presumed to have been generated by the model in previous
-     * interactions.
-     */
-    export interface Input {
-      /**
-       * Text inputs to the model - can contain template strings.
-       */
-      content: string | ResponsesAPI.ResponseInputText | Input.OutputText;
-
-      /**
-       * The role of the message input. One of `user`, `assistant`, `system`, or
-       * `developer`.
-       */
-      role: 'user' | 'assistant' | 'system' | 'developer';
-
-      /**
-       * The type of the message input. Always `message`.
-       */
-      type?: 'message';
-    }
-
-    export namespace Input {
-      /**
-       * A text output from the model.
-       */
-      export interface OutputText {
-        /**
-         * The text output from the model.
-         */
-        text: string;
-
-        /**
-         * The type of the output text. Always `output_text`.
-         */
-        type: 'output_text';
-      }
-    }
   }
 }
 
@@ -1306,10 +733,7 @@ Evals.RunListResponsesPage = RunListResponsesPage;
 export declare namespace Evals {
   export {
     type EvalCustomDataSourceConfig as EvalCustomDataSourceConfig,
-    type EvalLabelModelGrader as EvalLabelModelGrader,
     type EvalStoredCompletionsDataSourceConfig as EvalStoredCompletionsDataSourceConfig,
-    type EvalStringCheckGrader as EvalStringCheckGrader,
-    type EvalTextSimilarityGrader as EvalTextSimilarityGrader,
     type EvalCreateResponse as EvalCreateResponse,
     type EvalRetrieveResponse as EvalRetrieveResponse,
     type EvalUpdateResponse as EvalUpdateResponse,