openai 0.3.0 → 0.4.1

data/lib/openai/models/eval_create_params.rb

@@ -0,0 +1,451 @@
+# frozen_string_literal: true
+
+module OpenAI
+  module Models
+    # @see OpenAI::Resources::Evals#create
+    class EvalCreateParams < OpenAI::Internal::Type::BaseModel
+      extend OpenAI::Internal::Type::RequestParameters::Converter
+      include OpenAI::Internal::Type::RequestParameters
+
+      # @!attribute data_source_config
+      #   The configuration for the data source used for the evaluation runs. Dictates the
+      #   schema of the data used in the evaluation.
+      #
+      #   @return [OpenAI::Models::EvalCreateParams::DataSourceConfig::Custom, OpenAI::Models::EvalCreateParams::DataSourceConfig::Logs, OpenAI::Models::EvalCreateParams::DataSourceConfig::StoredCompletions]
+      required :data_source_config, union: -> { OpenAI::EvalCreateParams::DataSourceConfig }
+
+      # @!attribute testing_criteria
+      #   A list of graders for all eval runs in this group. Graders can reference
+      #   variables in the data source using double curly braces notation, like
+      #   `{{item.variable_name}}`. To reference the model's output, use the `sample`
+      #   namespace (ie, `{{sample.output_text}}`).
+      #
+      #   @return [Array<OpenAI::Models::EvalCreateParams::TestingCriterion::LabelModel, OpenAI::Models::Graders::StringCheckGrader, OpenAI::Models::EvalCreateParams::TestingCriterion::TextSimilarity, OpenAI::Models::EvalCreateParams::TestingCriterion::Python, OpenAI::Models::EvalCreateParams::TestingCriterion::ScoreModel>]
+      required :testing_criteria,
+               -> { OpenAI::Internal::Type::ArrayOf[union: OpenAI::EvalCreateParams::TestingCriterion] }
+
+      # @!attribute metadata
+      #   Set of 16 key-value pairs that can be attached to an object. This can be useful
+      #   for storing additional information about the object in a structured format, and
+      #   querying for objects via API or the dashboard.
+      #
+      #   Keys are strings with a maximum length of 64 characters. Values are strings with
+      #   a maximum length of 512 characters.
+      #
+      #   @return [Hash{Symbol=>String}, nil]
+      optional :metadata, OpenAI::Internal::Type::HashOf[String], nil?: true
+
+      # @!attribute name
+      #   The name of the evaluation.
+      #
+      #   @return [String, nil]
+      optional :name, String
+
+      # @!method initialize(data_source_config:, testing_criteria:, metadata: nil, name: nil, request_options: {})
+      #   Some parameter documentations has been truncated, see
+      #   {OpenAI::Models::EvalCreateParams} for more details.
+      #
+      #   @param data_source_config [OpenAI::Models::EvalCreateParams::DataSourceConfig::Custom, OpenAI::Models::EvalCreateParams::DataSourceConfig::Logs, OpenAI::Models::EvalCreateParams::DataSourceConfig::StoredCompletions] The configuration for the data source used for the evaluation runs. Dictates the
+      #
+      #   @param testing_criteria [Array<OpenAI::Models::EvalCreateParams::TestingCriterion::LabelModel, OpenAI::Models::Graders::StringCheckGrader, OpenAI::Models::EvalCreateParams::TestingCriterion::TextSimilarity, OpenAI::Models::EvalCreateParams::TestingCriterion::Python, OpenAI::Models::EvalCreateParams::TestingCriterion::ScoreModel>] A list of graders for all eval runs in this group. Graders can reference variabl
+      #
+      #   @param metadata [Hash{Symbol=>String}, nil] Set of 16 key-value pairs that can be attached to an object. This can be
+      #
+      #   @param name [String] The name of the evaluation.
+      #
+      #   @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}]
+
+      # The configuration for the data source used for the evaluation runs. Dictates the
+      # schema of the data used in the evaluation.
+      module DataSourceConfig
+        extend OpenAI::Internal::Type::Union
+
+        discriminator :type
+
+        # A CustomDataSourceConfig object that defines the schema for the data source used for the evaluation runs.
+        # This schema is used to define the shape of the data that will be:
+        # - Used to define your testing criteria and
+        # - What data is required when creating a run
+        variant :custom, -> { OpenAI::EvalCreateParams::DataSourceConfig::Custom }
+
+        # A data source config which specifies the metadata property of your logs query.
+        # This is usually metadata like `usecase=chatbot` or `prompt-version=v2`, etc.
+        variant :logs, -> { OpenAI::EvalCreateParams::DataSourceConfig::Logs }
+
+        # Deprecated in favor of LogsDataSourceConfig.
+        variant :stored_completions, -> { OpenAI::EvalCreateParams::DataSourceConfig::StoredCompletions }
+
+        class Custom < OpenAI::Internal::Type::BaseModel
+          # @!attribute item_schema
+          #   The json schema for each row in the data source.
+          #
+          #   @return [Hash{Symbol=>Object}]
+          required :item_schema, OpenAI::Internal::Type::HashOf[OpenAI::Internal::Type::Unknown]
+
+          # @!attribute type
+          #   The type of data source. Always `custom`.
+          #
+          #   @return [Symbol, :custom]
+          required :type, const: :custom
+
+          # @!attribute include_sample_schema
+          #   Whether the eval should expect you to populate the sample namespace (ie, by
+          #   generating responses off of your data source)
+          #
+          #   @return [Boolean, nil]
+          optional :include_sample_schema, OpenAI::Internal::Type::Boolean
+
+          # @!method initialize(item_schema:, include_sample_schema: nil, type: :custom)
+          #   Some parameter documentations has been truncated, see
+          #   {OpenAI::Models::EvalCreateParams::DataSourceConfig::Custom} for more details.
+          #
+          #   A CustomDataSourceConfig object that defines the schema for the data source used
+          #   for the evaluation runs. This schema is used to define the shape of the data
+          #   that will be:
+          #
+          #   - Used to define your testing criteria and
+          #   - What data is required when creating a run
+          #
+          #   @param item_schema [Hash{Symbol=>Object}] The json schema for each row in the data source.
+          #
+          #   @param include_sample_schema [Boolean] Whether the eval should expect you to populate the sample namespace (ie, by gene
+          #
+          #   @param type [Symbol, :custom] The type of data source. Always `custom`.
+        end
+
+        class Logs < OpenAI::Internal::Type::BaseModel
+          # @!attribute type
+          #   The type of data source. Always `logs`.
+          #
+          #   @return [Symbol, :logs]
+          required :type, const: :logs
+
+          # @!attribute metadata
+          #   Metadata filters for the logs data source.
+          #
+          #   @return [Hash{Symbol=>Object}, nil]
+          optional :metadata, OpenAI::Internal::Type::HashOf[OpenAI::Internal::Type::Unknown]
+
+          # @!method initialize(metadata: nil, type: :logs)
+          #   A data source config which specifies the metadata property of your logs query.
+          #   This is usually metadata like `usecase=chatbot` or `prompt-version=v2`, etc.
+          #
+          #   @param metadata [Hash{Symbol=>Object}] Metadata filters for the logs data source.
+          #
+          #   @param type [Symbol, :logs] The type of data source. Always `logs`.
+        end
+
+        # @deprecated
+        class StoredCompletions < OpenAI::Internal::Type::BaseModel
+          # @!attribute type
+          #   The type of data source. Always `stored_completions`.
+          #
+          #   @return [Symbol, :stored_completions]
+          required :type, const: :stored_completions
+
+          # @!attribute metadata
+          #   Metadata filters for the stored completions data source.
+          #
+          #   @return [Hash{Symbol=>Object}, nil]
+          optional :metadata, OpenAI::Internal::Type::HashOf[OpenAI::Internal::Type::Unknown]
+
+          # @!method initialize(metadata: nil, type: :stored_completions)
+          #   Deprecated in favor of LogsDataSourceConfig.
+          #
+          #   @param metadata [Hash{Symbol=>Object}] Metadata filters for the stored completions data source.
+          #
+          #   @param type [Symbol, :stored_completions] The type of data source. Always `stored_completions`.
+        end
+
+        # @!method self.variants
+        #   @return [Array(OpenAI::Models::EvalCreateParams::DataSourceConfig::Custom, OpenAI::Models::EvalCreateParams::DataSourceConfig::Logs, OpenAI::Models::EvalCreateParams::DataSourceConfig::StoredCompletions)]
+      end
+
+      # A LabelModelGrader object which uses a model to assign labels to each item in
+      # the evaluation.
+      module TestingCriterion
+        extend OpenAI::Internal::Type::Union
+
+        discriminator :type
+
+        # A LabelModelGrader object which uses a model to assign labels to each item
+        # in the evaluation.
+        variant :label_model, -> { OpenAI::EvalCreateParams::TestingCriterion::LabelModel }
+
+        # A StringCheckGrader object that performs a string comparison between input and reference using a specified operation.
+        variant :string_check, -> { OpenAI::Graders::StringCheckGrader }
+
+        # A TextSimilarityGrader object which grades text based on similarity metrics.
+        variant :text_similarity, -> { OpenAI::EvalCreateParams::TestingCriterion::TextSimilarity }
+
+        # A PythonGrader object that runs a python script on the input.
+        variant :python, -> { OpenAI::EvalCreateParams::TestingCriterion::Python }
+
+        # A ScoreModelGrader object that uses a model to assign a score to the input.
+        variant :score_model, -> { OpenAI::EvalCreateParams::TestingCriterion::ScoreModel }
+
+        class LabelModel < OpenAI::Internal::Type::BaseModel
+          # @!attribute input
+          #   A list of chat messages forming the prompt or context. May include variable
+          #   references to the `item` namespace, ie {{item.name}}.
+          #
+          #   @return [Array<OpenAI::Models::EvalCreateParams::TestingCriterion::LabelModel::Input::SimpleInputMessage, OpenAI::Models::EvalCreateParams::TestingCriterion::LabelModel::Input::EvalItem>]
+          required :input,
+                   -> {
+                     OpenAI::Internal::Type::ArrayOf[union: OpenAI::EvalCreateParams::TestingCriterion::LabelModel::Input]
+                   }
+
+          # @!attribute labels
+          #   The labels to classify to each item in the evaluation.
+          #
+          #   @return [Array<String>]
+          required :labels, OpenAI::Internal::Type::ArrayOf[String]
+
+          # @!attribute model
+          #   The model to use for the evaluation. Must support structured outputs.
+          #
+          #   @return [String]
+          required :model, String
+
+          # @!attribute name
+          #   The name of the grader.
+          #
+          #   @return [String]
+          required :name, String
+
+          # @!attribute passing_labels
+          #   The labels that indicate a passing result. Must be a subset of labels.
+          #
+          #   @return [Array<String>]
+          required :passing_labels, OpenAI::Internal::Type::ArrayOf[String]
+
+          # @!attribute type
+          #   The object type, which is always `label_model`.
+          #
+          #   @return [Symbol, :label_model]
+          required :type, const: :label_model
+
+          # @!method initialize(input:, labels:, model:, name:, passing_labels:, type: :label_model)
+          #   Some parameter documentations has been truncated, see
+          #   {OpenAI::Models::EvalCreateParams::TestingCriterion::LabelModel} for more
+          #   details.
+          #
+          #   A LabelModelGrader object which uses a model to assign labels to each item in
+          #   the evaluation.
+          #
+          #   @param input [Array<OpenAI::Models::EvalCreateParams::TestingCriterion::LabelModel::Input::SimpleInputMessage, OpenAI::Models::EvalCreateParams::TestingCriterion::LabelModel::Input::EvalItem>] A list of chat messages forming the prompt or context. May include variable refe
+          #
+          #   @param labels [Array<String>] The labels to classify to each item in the evaluation.
+          #
+          #   @param model [String] The model to use for the evaluation. Must support structured outputs.
+          #
+          #   @param name [String] The name of the grader.
+          #
+          #   @param passing_labels [Array<String>] The labels that indicate a passing result. Must be a subset of labels.
+          #
+          #   @param type [Symbol, :label_model] The object type, which is always `label_model`.
+
+          # A chat message that makes up the prompt or context. May include variable
+          # references to the `item` namespace, ie {{item.name}}.
+          module Input
+            extend OpenAI::Internal::Type::Union
+
+            variant -> { OpenAI::EvalCreateParams::TestingCriterion::LabelModel::Input::SimpleInputMessage }
+
+            # A message input to the model with a role indicating instruction following
+            # hierarchy. Instructions given with the `developer` or `system` role take
+            # precedence over instructions given with the `user` role. Messages with the
+            # `assistant` role are presumed to have been generated by the model in previous
+            # interactions.
+            variant -> { OpenAI::EvalCreateParams::TestingCriterion::LabelModel::Input::EvalItem }
+
+            class SimpleInputMessage < OpenAI::Internal::Type::BaseModel
+              # @!attribute content
+              #   The content of the message.
+              #
+              #   @return [String]
+              required :content, String
+
+              # @!attribute role
+              #   The role of the message (e.g. "system", "assistant", "user").
+              #
+              #   @return [String]
+              required :role, String
+
+              # @!method initialize(content:, role:)
+              #   @param content [String] The content of the message.
+              #
+              #   @param role [String] The role of the message (e.g. "system", "assistant", "user").
+            end
+
+            class EvalItem < OpenAI::Internal::Type::BaseModel
+              # @!attribute content
+              #   Text inputs to the model - can contain template strings.
+              #
+              #   @return [String, OpenAI::Models::Responses::ResponseInputText, OpenAI::Models::EvalCreateParams::TestingCriterion::LabelModel::Input::EvalItem::Content::OutputText]
+              required :content,
+                       union: -> {
+                         OpenAI::EvalCreateParams::TestingCriterion::LabelModel::Input::EvalItem::Content
+                       }
+
+              # @!attribute role
+              #   The role of the message input. One of `user`, `assistant`, `system`, or
+              #   `developer`.
+              #
+              #   @return [Symbol, OpenAI::Models::EvalCreateParams::TestingCriterion::LabelModel::Input::EvalItem::Role]
+              required :role,
+                       enum: -> {
+                         OpenAI::EvalCreateParams::TestingCriterion::LabelModel::Input::EvalItem::Role
+                       }
+
+              # @!attribute type
+              #   The type of the message input. Always `message`.
+              #
+              #   @return [Symbol, OpenAI::Models::EvalCreateParams::TestingCriterion::LabelModel::Input::EvalItem::Type, nil]
+              optional :type,
+                       enum: -> {
+                         OpenAI::EvalCreateParams::TestingCriterion::LabelModel::Input::EvalItem::Type
+                       }
+
+              # @!method initialize(content:, role:, type: nil)
+              #   Some parameter documentations has been truncated, see
+              #   {OpenAI::Models::EvalCreateParams::TestingCriterion::LabelModel::Input::EvalItem}
+              #   for more details.
+              #
+              #   A message input to the model with a role indicating instruction following
+              #   hierarchy. Instructions given with the `developer` or `system` role take
+              #   precedence over instructions given with the `user` role. Messages with the
+              #   `assistant` role are presumed to have been generated by the model in previous
+              #   interactions.
+              #
+              #   @param content [String, OpenAI::Models::Responses::ResponseInputText, OpenAI::Models::EvalCreateParams::TestingCriterion::LabelModel::Input::EvalItem::Content::OutputText] Text inputs to the model - can contain template strings.
+              #
+              #   @param role [Symbol, OpenAI::Models::EvalCreateParams::TestingCriterion::LabelModel::Input::EvalItem::Role] The role of the message input. One of `user`, `assistant`, `system`, or
+              #
+              #   @param type [Symbol, OpenAI::Models::EvalCreateParams::TestingCriterion::LabelModel::Input::EvalItem::Type] The type of the message input. Always `message`.
+
+              # Text inputs to the model - can contain template strings.
+              #
+              # @see OpenAI::Models::EvalCreateParams::TestingCriterion::LabelModel::Input::EvalItem#content
+              module Content
+                extend OpenAI::Internal::Type::Union
+
+                # A text input to the model.
+                variant String
+
+                # A text input to the model.
+                variant -> { OpenAI::Responses::ResponseInputText }
+
+                # A text output from the model.
+                variant -> {
+                  OpenAI::EvalCreateParams::TestingCriterion::LabelModel::Input::EvalItem::Content::OutputText
+                }
+
+                class OutputText < OpenAI::Internal::Type::BaseModel
+                  # @!attribute text
+                  #   The text output from the model.
+                  #
+                  #   @return [String]
+                  required :text, String
+
+                  # @!attribute type
+                  #   The type of the output text. Always `output_text`.
+                  #
+                  #   @return [Symbol, :output_text]
+                  required :type, const: :output_text
+
+                  # @!method initialize(text:, type: :output_text)
+                  #   Some parameter documentations has been truncated, see
+                  #   {OpenAI::Models::EvalCreateParams::TestingCriterion::LabelModel::Input::EvalItem::Content::OutputText}
+                  #   for more details.
+                  #
+                  #   A text output from the model.
+                  #
+                  #   @param text [String] The text output from the model.
+                  #
+                  #   @param type [Symbol, :output_text] The type of the output text. Always `output_text`.
+                end
+
+                # @!method self.variants
+                #   @return [Array(String, OpenAI::Models::Responses::ResponseInputText, OpenAI::Models::EvalCreateParams::TestingCriterion::LabelModel::Input::EvalItem::Content::OutputText)]
+              end
+
+              # The role of the message input. One of `user`, `assistant`, `system`, or
+              # `developer`.
+              #
+              # @see OpenAI::Models::EvalCreateParams::TestingCriterion::LabelModel::Input::EvalItem#role
+              module Role
+                extend OpenAI::Internal::Type::Enum
+
+                USER = :user
+                ASSISTANT = :assistant
+                SYSTEM = :system
+                DEVELOPER = :developer
+
+                # @!method self.values
+                #   @return [Array<Symbol>]
+              end
+
+              # The type of the message input. Always `message`.
+              #
+              # @see OpenAI::Models::EvalCreateParams::TestingCriterion::LabelModel::Input::EvalItem#type
+              module Type
+                extend OpenAI::Internal::Type::Enum
+
+                MESSAGE = :message
+
+                # @!method self.values
+                #   @return [Array<Symbol>]
+              end
+            end
+
+            # @!method self.variants
+            #   @return [Array(OpenAI::Models::EvalCreateParams::TestingCriterion::LabelModel::Input::SimpleInputMessage, OpenAI::Models::EvalCreateParams::TestingCriterion::LabelModel::Input::EvalItem)]
+          end
+        end
+
+        class TextSimilarity < OpenAI::Models::Graders::TextSimilarityGrader
+          # @!attribute pass_threshold
+          #   The threshold for the score.
+          #
+          #   @return [Float]
+          required :pass_threshold, Float
+
+          # @!method initialize(pass_threshold:)
+          #   A TextSimilarityGrader object which grades text based on similarity metrics.
+          #
+          #   @param pass_threshold [Float] The threshold for the score.
+        end
+
+        class Python < OpenAI::Models::Graders::PythonGrader
+          # @!attribute pass_threshold
+          #   The threshold for the score.
+          #
+          #   @return [Float, nil]
+          optional :pass_threshold, Float
+
+          # @!method initialize(pass_threshold: nil)
+          #   A PythonGrader object that runs a python script on the input.
+          #
+          #   @param pass_threshold [Float] The threshold for the score.
+        end
+
+        class ScoreModel < OpenAI::Models::Graders::ScoreModelGrader
+          # @!attribute pass_threshold
+          #   The threshold for the score.
+          #
+          #   @return [Float, nil]
+          optional :pass_threshold, Float
+
+          # @!method initialize(pass_threshold: nil)
+          #   A ScoreModelGrader object that uses a model to assign a score to the input.
+          #
+          #   @param pass_threshold [Float] The threshold for the score.
+        end
+
+        # @!method self.variants
+        #   @return [Array(OpenAI::Models::EvalCreateParams::TestingCriterion::LabelModel, OpenAI::Models::Graders::StringCheckGrader, OpenAI::Models::EvalCreateParams::TestingCriterion::TextSimilarity, OpenAI::Models::EvalCreateParams::TestingCriterion::Python, OpenAI::Models::EvalCreateParams::TestingCriterion::ScoreModel)]
+      end
+    end
+  end
+end

data/lib/openai/models/eval_create_response.rb

@@ -0,0 +1,214 @@
+# frozen_string_literal: true
+
+module OpenAI
+  module Models
+    # @see OpenAI::Resources::Evals#create
+    class EvalCreateResponse < OpenAI::Internal::Type::BaseModel
+      # @!attribute id
+      #   Unique identifier for the evaluation.
+      #
+      #   @return [String]
+      required :id, String
+
+      # @!attribute created_at
+      #   The Unix timestamp (in seconds) for when the eval was created.
+      #
+      #   @return [Integer]
+      required :created_at, Integer
+
+      # @!attribute data_source_config
+      #   Configuration of data sources used in runs of the evaluation.
+      #
+      #   @return [OpenAI::Models::EvalCustomDataSourceConfig, OpenAI::Models::EvalCreateResponse::DataSourceConfig::Logs, OpenAI::Models::EvalStoredCompletionsDataSourceConfig]
+      required :data_source_config, union: -> { OpenAI::Models::EvalCreateResponse::DataSourceConfig }
+
+      # @!attribute metadata
+      #   Set of 16 key-value pairs that can be attached to an object. This can be useful
+      #   for storing additional information about the object in a structured format, and
+      #   querying for objects via API or the dashboard.
+      #
+      #   Keys are strings with a maximum length of 64 characters. Values are strings with
+      #   a maximum length of 512 characters.
+      #
+      #   @return [Hash{Symbol=>String}, nil]
+      required :metadata, OpenAI::Internal::Type::HashOf[String], nil?: true
+
+      # @!attribute name
+      #   The name of the evaluation.
+      #
+      #   @return [String]
+      required :name, String
+
+      # @!attribute object
+      #   The object type.
+      #
+      #   @return [Symbol, :eval]
+      required :object, const: :eval
+
+      # @!attribute testing_criteria
+      #   A list of testing criteria.
+      #
+      #   @return [Array<OpenAI::Models::Graders::LabelModelGrader, OpenAI::Models::Graders::StringCheckGrader, OpenAI::Models::EvalCreateResponse::TestingCriterion::EvalGraderTextSimilarity, OpenAI::Models::EvalCreateResponse::TestingCriterion::EvalGraderPython, OpenAI::Models::EvalCreateResponse::TestingCriterion::EvalGraderScoreModel>]
+      required :testing_criteria,
+               -> { OpenAI::Internal::Type::ArrayOf[union: OpenAI::Models::EvalCreateResponse::TestingCriterion] }
+
+      # @!method initialize(id:, created_at:, data_source_config:, metadata:, name:, testing_criteria:, object: :eval)
+      #   Some parameter documentations has been truncated, see
+      #   {OpenAI::Models::EvalCreateResponse} for more details.
+      #
+      #   An Eval object with a data source config and testing criteria. An Eval
+      #   represents a task to be done for your LLM integration. Like:
+      #
+      #   - Improve the quality of my chatbot
+      #   - See how well my chatbot handles customer support
+      #   - Check if o4-mini is better at my usecase than gpt-4o
+      #
+      #   @param id [String] Unique identifier for the evaluation.
+      #
+      #   @param created_at [Integer] The Unix timestamp (in seconds) for when the eval was created.
+      #
+      #   @param data_source_config [OpenAI::Models::EvalCustomDataSourceConfig, OpenAI::Models::EvalCreateResponse::DataSourceConfig::Logs, OpenAI::Models::EvalStoredCompletionsDataSourceConfig] Configuration of data sources used in runs of the evaluation.
+      #
+      #   @param metadata [Hash{Symbol=>String}, nil] Set of 16 key-value pairs that can be attached to an object. This can be
+      #
+      #   @param name [String] The name of the evaluation.
+      #
+      #   @param testing_criteria [Array<OpenAI::Models::Graders::LabelModelGrader, OpenAI::Models::Graders::StringCheckGrader, OpenAI::Models::EvalCreateResponse::TestingCriterion::EvalGraderTextSimilarity, OpenAI::Models::EvalCreateResponse::TestingCriterion::EvalGraderPython, OpenAI::Models::EvalCreateResponse::TestingCriterion::EvalGraderScoreModel>] A list of testing criteria.
+      #
+      #   @param object [Symbol, :eval] The object type.
+
+      # Configuration of data sources used in runs of the evaluation.
+      #
+      # @see OpenAI::Models::EvalCreateResponse#data_source_config
+      module DataSourceConfig
+        extend OpenAI::Internal::Type::Union
+
+        discriminator :type
+
+        # A CustomDataSourceConfig which specifies the schema of your `item` and optionally `sample` namespaces.
+        # The response schema defines the shape of the data that will be:
+        # - Used to define your testing criteria and
+        # - What data is required when creating a run
+        variant :custom, -> { OpenAI::EvalCustomDataSourceConfig }
+
+        # A LogsDataSourceConfig which specifies the metadata property of your logs query.
+        # This is usually metadata like `usecase=chatbot` or `prompt-version=v2`, etc.
+        # The schema returned by this data source config is used to defined what variables are available in your evals.
+        # `item` and `sample` are both defined when using this data source config.
+        variant :logs, -> { OpenAI::Models::EvalCreateResponse::DataSourceConfig::Logs }
+
+        # Deprecated in favor of LogsDataSourceConfig.
+        variant :stored_completions, -> { OpenAI::EvalStoredCompletionsDataSourceConfig }
+
+        class Logs < OpenAI::Internal::Type::BaseModel
+          # @!attribute schema
+          #   The json schema for the run data source items. Learn how to build JSON schemas
+          #   [here](https://json-schema.org/).
+          #
+          #   @return [Hash{Symbol=>Object}]
+          required :schema, OpenAI::Internal::Type::HashOf[OpenAI::Internal::Type::Unknown]
+
+          # @!attribute type
+          #   The type of data source. Always `logs`.
+          #
+          #   @return [Symbol, :logs]
+          required :type, const: :logs
+
+          # @!attribute metadata
+          #   Set of 16 key-value pairs that can be attached to an object. This can be useful
+          #   for storing additional information about the object in a structured format, and
+          #   querying for objects via API or the dashboard.
+          #
+          #   Keys are strings with a maximum length of 64 characters. Values are strings with
+          #   a maximum length of 512 characters.
+          #
+          #   @return [Hash{Symbol=>String}, nil]
+          optional :metadata, OpenAI::Internal::Type::HashOf[String], nil?: true
+
+          # @!method initialize(schema:, metadata: nil, type: :logs)
+          #   Some parameter documentations has been truncated, see
+          #   {OpenAI::Models::EvalCreateResponse::DataSourceConfig::Logs} for more details.
+          #
+          #   A LogsDataSourceConfig which specifies the metadata property of your logs query.
+          #   This is usually metadata like `usecase=chatbot` or `prompt-version=v2`, etc. The
+          #   schema returned by this data source config is used to defined what variables are
+          #   available in your evals. `item` and `sample` are both defined when using this
+          #   data source config.
+          #
+          #   @param schema [Hash{Symbol=>Object}] The json schema for the run data source items.
+          #
+          #   @param metadata [Hash{Symbol=>String}, nil] Set of 16 key-value pairs that can be attached to an object. This can be
+          #
+          #   @param type [Symbol, :logs] The type of data source. Always `logs`.
+        end
+
+        # @!method self.variants
+        #   @return [Array(OpenAI::Models::EvalCustomDataSourceConfig, OpenAI::Models::EvalCreateResponse::DataSourceConfig::Logs, OpenAI::Models::EvalStoredCompletionsDataSourceConfig)]
+      end
+
+      # A LabelModelGrader object which uses a model to assign labels to each item in
+      # the evaluation.
+      module TestingCriterion
+        extend OpenAI::Internal::Type::Union
+
+        # A LabelModelGrader object which uses a model to assign labels to each item
+        # in the evaluation.
+        variant -> { OpenAI::Graders::LabelModelGrader }
+
+        # A StringCheckGrader object that performs a string comparison between input and reference using a specified operation.
+        variant -> { OpenAI::Graders::StringCheckGrader }
+
+        # A TextSimilarityGrader object which grades text based on similarity metrics.
+        variant -> { OpenAI::Models::EvalCreateResponse::TestingCriterion::EvalGraderTextSimilarity }
+
+        # A PythonGrader object that runs a python script on the input.
+        variant -> { OpenAI::Models::EvalCreateResponse::TestingCriterion::EvalGraderPython }
+
+        # A ScoreModelGrader object that uses a model to assign a score to the input.
+        variant -> { OpenAI::Models::EvalCreateResponse::TestingCriterion::EvalGraderScoreModel }
+
+        class EvalGraderTextSimilarity < OpenAI::Models::Graders::TextSimilarityGrader
+          # @!attribute pass_threshold
+          #   The threshold for the score.
+          #
+          #   @return [Float]
+          required :pass_threshold, Float
+
+          # @!method initialize(pass_threshold:)
+          #   A TextSimilarityGrader object which grades text based on similarity metrics.
+          #
+          #   @param pass_threshold [Float] The threshold for the score.
+        end
+
+        class EvalGraderPython < OpenAI::Models::Graders::PythonGrader
+          # @!attribute pass_threshold
+          #   The threshold for the score.
+          #
+          #   @return [Float, nil]
+          optional :pass_threshold, Float
+
+          # @!method initialize(pass_threshold: nil)
+          #   A PythonGrader object that runs a python script on the input.
+          #
+          #   @param pass_threshold [Float] The threshold for the score.
+        end
+
+        class EvalGraderScoreModel < OpenAI::Models::Graders::ScoreModelGrader
+          # @!attribute pass_threshold
+          #   The threshold for the score.
+          #
+          #   @return [Float, nil]
+          optional :pass_threshold, Float
+
+          # @!method initialize(pass_threshold: nil)
+          #   A ScoreModelGrader object that uses a model to assign a score to the input.
+          #
+          #   @param pass_threshold [Float] The threshold for the score.
+        end
+
+        # @!method self.variants
+        #   @return [Array(OpenAI::Models::Graders::LabelModelGrader, OpenAI::Models::Graders::StringCheckGrader, OpenAI::Models::EvalCreateResponse::TestingCriterion::EvalGraderTextSimilarity, OpenAI::Models::EvalCreateResponse::TestingCriterion::EvalGraderPython, OpenAI::Models::EvalCreateResponse::TestingCriterion::EvalGraderScoreModel)]
+      end
+    end
+  end
+end