openai 0.3.0 → 0.4.0.pre.beta.1

data/rbi/openai/models/eval_create_response.rbi

@@ -0,0 +1,330 @@
+# typed: strong
+
+module OpenAI
+  module Models
+    class EvalCreateResponse < OpenAI::Internal::Type::BaseModel
+      OrHash =
+        T.type_alias do
+          T.any(OpenAI::Models::EvalCreateResponse, OpenAI::Internal::AnyHash)
+        end
+
+      # Unique identifier for the evaluation.
+      sig { returns(String) }
+      attr_accessor :id
+
+      # The Unix timestamp (in seconds) for when the eval was created.
+      sig { returns(Integer) }
+      attr_accessor :created_at
+
+      # Configuration of data sources used in runs of the evaluation.
+      sig do
+        returns(OpenAI::Models::EvalCreateResponse::DataSourceConfig::Variants)
+      end
+      attr_accessor :data_source_config
+
+      # Set of 16 key-value pairs that can be attached to an object. This can be useful
+      # for storing additional information about the object in a structured format, and
+      # querying for objects via API or the dashboard.
+      #
+      # Keys are strings with a maximum length of 64 characters. Values are strings with
+      # a maximum length of 512 characters.
+      sig { returns(T.nilable(T::Hash[Symbol, String])) }
+      attr_accessor :metadata
+
+      # The name of the evaluation.
+      sig { returns(String) }
+      attr_accessor :name
+
+      # The object type.
+      sig { returns(Symbol) }
+      attr_accessor :object
+
+      # A list of testing criteria.
+      sig do
+        returns(
+          T::Array[
+            OpenAI::Models::EvalCreateResponse::TestingCriterion::Variants
+          ]
+        )
+      end
+      attr_accessor :testing_criteria
+
+      # An Eval object with a data source config and testing criteria. An Eval
+      # represents a task to be done for your LLM integration. Like:
+      #
+      # - Improve the quality of my chatbot
+      # - See how well my chatbot handles customer support
+      # - Check if o4-mini is better at my usecase than gpt-4o
+      sig do
+        params(
+          id: String,
+          created_at: Integer,
+          data_source_config:
+            T.any(
+              OpenAI::EvalCustomDataSourceConfig::OrHash,
+              OpenAI::Models::EvalCreateResponse::DataSourceConfig::Logs::OrHash,
+              OpenAI::EvalStoredCompletionsDataSourceConfig::OrHash
+            ),
+          metadata: T.nilable(T::Hash[Symbol, String]),
+          name: String,
+          testing_criteria:
+            T::Array[
+              T.any(
+                OpenAI::Models::Graders::LabelModelGrader::OrHash,
+                OpenAI::Models::Graders::StringCheckGrader::OrHash,
+                OpenAI::Models::EvalCreateResponse::TestingCriterion::EvalGraderTextSimilarity::OrHash,
+                OpenAI::Models::EvalCreateResponse::TestingCriterion::EvalGraderPython::OrHash,
+                OpenAI::Models::EvalCreateResponse::TestingCriterion::EvalGraderScoreModel::OrHash
+              )
+            ],
+          object: Symbol
+        ).returns(T.attached_class)
+      end
+      def self.new(
+        # Unique identifier for the evaluation.
+        id:,
+        # The Unix timestamp (in seconds) for when the eval was created.
+        created_at:,
+        # Configuration of data sources used in runs of the evaluation.
+        data_source_config:,
+        # Set of 16 key-value pairs that can be attached to an object. This can be useful
+        # for storing additional information about the object in a structured format, and
+        # querying for objects via API or the dashboard.
+        #
+        # Keys are strings with a maximum length of 64 characters. Values are strings with
+        # a maximum length of 512 characters.
+        metadata:,
+        # The name of the evaluation.
+        name:,
+        # A list of testing criteria.
+        testing_criteria:,
+        # The object type.
+        object: :eval
+      )
+      end
+
+      sig do
+        override.returns(
+          {
+            id: String,
+            created_at: Integer,
+            data_source_config:
+              OpenAI::Models::EvalCreateResponse::DataSourceConfig::Variants,
+            metadata: T.nilable(T::Hash[Symbol, String]),
+            name: String,
+            object: Symbol,
+            testing_criteria:
+              T::Array[
+                OpenAI::Models::EvalCreateResponse::TestingCriterion::Variants
+              ]
+          }
+        )
+      end
+      def to_hash
+      end
+
+      # Configuration of data sources used in runs of the evaluation.
+      module DataSourceConfig
+        extend OpenAI::Internal::Type::Union
+
+        Variants =
+          T.type_alias do
+            T.any(
+              OpenAI::EvalCustomDataSourceConfig,
+              OpenAI::Models::EvalCreateResponse::DataSourceConfig::Logs,
+              OpenAI::EvalStoredCompletionsDataSourceConfig
+            )
+          end
+
+        class Logs < OpenAI::Internal::Type::BaseModel
+          OrHash =
+            T.type_alias do
+              T.any(
+                OpenAI::Models::EvalCreateResponse::DataSourceConfig::Logs,
+                OpenAI::Internal::AnyHash
+              )
+            end
+
+          # The json schema for the run data source items. Learn how to build JSON schemas
+          # [here](https://json-schema.org/).
+          sig { returns(T::Hash[Symbol, T.anything]) }
+          attr_accessor :schema
+
+          # The type of data source. Always `logs`.
+          sig { returns(Symbol) }
+          attr_accessor :type
+
+          # Set of 16 key-value pairs that can be attached to an object. This can be useful
+          # for storing additional information about the object in a structured format, and
+          # querying for objects via API or the dashboard.
+          #
+          # Keys are strings with a maximum length of 64 characters. Values are strings with
+          # a maximum length of 512 characters.
+          sig { returns(T.nilable(T::Hash[Symbol, String])) }
+          attr_accessor :metadata
+
+          # A LogsDataSourceConfig which specifies the metadata property of your logs query.
+          # This is usually metadata like `usecase=chatbot` or `prompt-version=v2`, etc. The
+          # schema returned by this data source config is used to defined what variables are
+          # available in your evals. `item` and `sample` are both defined when using this
+          # data source config.
+          sig do
+            params(
+              schema: T::Hash[Symbol, T.anything],
+              metadata: T.nilable(T::Hash[Symbol, String]),
+              type: Symbol
+            ).returns(T.attached_class)
+          end
+          def self.new(
+            # The json schema for the run data source items. Learn how to build JSON schemas
+            # [here](https://json-schema.org/).
+            schema:,
+            # Set of 16 key-value pairs that can be attached to an object. This can be useful
+            # for storing additional information about the object in a structured format, and
+            # querying for objects via API or the dashboard.
+            #
+            # Keys are strings with a maximum length of 64 characters. Values are strings with
+            # a maximum length of 512 characters.
+            metadata: nil,
+            # The type of data source. Always `logs`.
+            type: :logs
+          )
+          end
+
+          sig do
+            override.returns(
+              {
+                schema: T::Hash[Symbol, T.anything],
+                type: Symbol,
+                metadata: T.nilable(T::Hash[Symbol, String])
+              }
+            )
+          end
+          def to_hash
+          end
+        end
+
+        sig do
+          override.returns(
+            T::Array[
+              OpenAI::Models::EvalCreateResponse::DataSourceConfig::Variants
+            ]
+          )
+        end
+        def self.variants
+        end
+      end
+
+      # A LabelModelGrader object which uses a model to assign labels to each item in
+      # the evaluation.
+      module TestingCriterion
+        extend OpenAI::Internal::Type::Union
+
+        Variants =
+          T.type_alias do
+            T.any(
+              OpenAI::Models::Graders::LabelModelGrader,
+              OpenAI::Models::Graders::StringCheckGrader,
+              OpenAI::Models::EvalCreateResponse::TestingCriterion::EvalGraderTextSimilarity,
+              OpenAI::Models::EvalCreateResponse::TestingCriterion::EvalGraderPython,
+              OpenAI::Models::EvalCreateResponse::TestingCriterion::EvalGraderScoreModel
+            )
+          end
+
+        class EvalGraderTextSimilarity < OpenAI::Models::Graders::TextSimilarityGrader
+          OrHash =
+            T.type_alias do
+              T.any(
+                OpenAI::Models::EvalCreateResponse::TestingCriterion::EvalGraderTextSimilarity,
+                OpenAI::Internal::AnyHash
+              )
+            end
+
+          # The threshold for the score.
+          sig { returns(Float) }
+          attr_accessor :pass_threshold
+
+          # A TextSimilarityGrader object which grades text based on similarity metrics.
+          sig { params(pass_threshold: Float).returns(T.attached_class) }
+          def self.new(
+            # The threshold for the score.
+            pass_threshold:
+          )
+          end
+
+          sig { override.returns({ pass_threshold: Float }) }
+          def to_hash
+          end
+        end
+
+        class EvalGraderPython < OpenAI::Models::Graders::PythonGrader
+          OrHash =
+            T.type_alias do
+              T.any(
+                OpenAI::Models::EvalCreateResponse::TestingCriterion::EvalGraderPython,
+                OpenAI::Internal::AnyHash
+              )
+            end
+
+          # The threshold for the score.
+          sig { returns(T.nilable(Float)) }
+          attr_reader :pass_threshold
+
+          sig { params(pass_threshold: Float).void }
+          attr_writer :pass_threshold
+
+          # A PythonGrader object that runs a python script on the input.
+          sig { params(pass_threshold: Float).returns(T.attached_class) }
+          def self.new(
+            # The threshold for the score.
+            pass_threshold: nil
+          )
+          end
+
+          sig { override.returns({ pass_threshold: Float }) }
+          def to_hash
+          end
+        end
+
+        class EvalGraderScoreModel < OpenAI::Models::Graders::ScoreModelGrader
+          OrHash =
+            T.type_alias do
+              T.any(
+                OpenAI::Models::EvalCreateResponse::TestingCriterion::EvalGraderScoreModel,
+                OpenAI::Internal::AnyHash
+              )
+            end
+
+          # The threshold for the score.
+          sig { returns(T.nilable(Float)) }
+          attr_reader :pass_threshold
+
+          sig { params(pass_threshold: Float).void }
+          attr_writer :pass_threshold
+
+          # A ScoreModelGrader object that uses a model to assign a score to the input.
+          sig { params(pass_threshold: Float).returns(T.attached_class) }
+          def self.new(
+            # The threshold for the score.
+            pass_threshold: nil
+          )
+          end
+
+          sig { override.returns({ pass_threshold: Float }) }
+          def to_hash
+          end
+        end
+
+        sig do
+          override.returns(
+            T::Array[
+              OpenAI::Models::EvalCreateResponse::TestingCriterion::Variants
+            ]
+          )
+        end
+        def self.variants
+        end
+      end
+    end
+  end
+end

data/rbi/openai/models/eval_custom_data_source_config.rbi

@@ -0,0 +1,47 @@
+# typed: strong
+
+module OpenAI
+  module Models
+    class EvalCustomDataSourceConfig < OpenAI::Internal::Type::BaseModel
+      OrHash =
+        T.type_alias do
+          T.any(OpenAI::EvalCustomDataSourceConfig, OpenAI::Internal::AnyHash)
+        end
+
+      # The json schema for the run data source items. Learn how to build JSON schemas
+      # [here](https://json-schema.org/).
+      sig { returns(T::Hash[Symbol, T.anything]) }
+      attr_accessor :schema
+
+      # The type of data source. Always `custom`.
+      sig { returns(Symbol) }
+      attr_accessor :type
+
+      # A CustomDataSourceConfig which specifies the schema of your `item` and
+      # optionally `sample` namespaces. The response schema defines the shape of the
+      # data that will be:
+      #
+      # - Used to define your testing criteria and
+      # - What data is required when creating a run
+      sig do
+        params(schema: T::Hash[Symbol, T.anything], type: Symbol).returns(
+          T.attached_class
+        )
+      end
+      def self.new(
+        # The json schema for the run data source items. Learn how to build JSON schemas
+        # [here](https://json-schema.org/).
+        schema:,
+        # The type of data source. Always `custom`.
+        type: :custom
+      )
+      end
+
+      sig do
+        override.returns({ schema: T::Hash[Symbol, T.anything], type: Symbol })
+      end
+      def to_hash
+      end
+    end
+  end
+end

data/rbi/openai/models/eval_delete_params.rbi

@@ -0,0 +1,27 @@
+# typed: strong
+
+module OpenAI
+  module Models
+    class EvalDeleteParams < OpenAI::Internal::Type::BaseModel
+      extend OpenAI::Internal::Type::RequestParameters::Converter
+      include OpenAI::Internal::Type::RequestParameters
+
+      OrHash =
+        T.type_alias do
+          T.any(OpenAI::EvalDeleteParams, OpenAI::Internal::AnyHash)
+        end
+
+      sig do
+        params(request_options: OpenAI::RequestOptions::OrHash).returns(
+          T.attached_class
+        )
+      end
+      def self.new(request_options: {})
+      end
+
+      sig { override.returns({ request_options: OpenAI::RequestOptions }) }
+      def to_hash
+      end
+    end
+  end
+end

data/rbi/openai/models/eval_delete_response.rbi

@@ -0,0 +1,37 @@
+# typed: strong
+
+module OpenAI
+  module Models
+    class EvalDeleteResponse < OpenAI::Internal::Type::BaseModel
+      OrHash =
+        T.type_alias do
+          T.any(OpenAI::Models::EvalDeleteResponse, OpenAI::Internal::AnyHash)
+        end
+
+      sig { returns(T::Boolean) }
+      attr_accessor :deleted
+
+      sig { returns(String) }
+      attr_accessor :eval_id
+
+      sig { returns(String) }
+      attr_accessor :object
+
+      sig do
+        params(deleted: T::Boolean, eval_id: String, object: String).returns(
+          T.attached_class
+        )
+      end
+      def self.new(deleted:, eval_id:, object:)
+      end
+
+      sig do
+        override.returns(
+          { deleted: T::Boolean, eval_id: String, object: String }
+        )
+      end
+      def to_hash
+      end
+    end
+  end
+end

data/rbi/openai/models/eval_list_params.rbi

@@ -0,0 +1,127 @@
+# typed: strong
+
+module OpenAI
+  module Models
+    class EvalListParams < OpenAI::Internal::Type::BaseModel
+      extend OpenAI::Internal::Type::RequestParameters::Converter
+      include OpenAI::Internal::Type::RequestParameters
+
+      OrHash =
+        T.type_alias do
+          T.any(OpenAI::EvalListParams, OpenAI::Internal::AnyHash)
+        end
+
+      # Identifier for the last eval from the previous pagination request.
+      sig { returns(T.nilable(String)) }
+      attr_reader :after
+
+      sig { params(after: String).void }
+      attr_writer :after
+
+      # Number of evals to retrieve.
+      sig { returns(T.nilable(Integer)) }
+      attr_reader :limit
+
+      sig { params(limit: Integer).void }
+      attr_writer :limit
+
+      # Sort order for evals by timestamp. Use `asc` for ascending order or `desc` for
+      # descending order.
+      sig { returns(T.nilable(OpenAI::EvalListParams::Order::OrSymbol)) }
+      attr_reader :order
+
+      sig { params(order: OpenAI::EvalListParams::Order::OrSymbol).void }
+      attr_writer :order
+
+      # Evals can be ordered by creation time or last updated time. Use `created_at` for
+      # creation time or `updated_at` for last updated time.
+      sig { returns(T.nilable(OpenAI::EvalListParams::OrderBy::OrSymbol)) }
+      attr_reader :order_by
+
+      sig { params(order_by: OpenAI::EvalListParams::OrderBy::OrSymbol).void }
+      attr_writer :order_by
+
+      sig do
+        params(
+          after: String,
+          limit: Integer,
+          order: OpenAI::EvalListParams::Order::OrSymbol,
+          order_by: OpenAI::EvalListParams::OrderBy::OrSymbol,
+          request_options: OpenAI::RequestOptions::OrHash
+        ).returns(T.attached_class)
+      end
+      def self.new(
+        # Identifier for the last eval from the previous pagination request.
+        after: nil,
+        # Number of evals to retrieve.
+        limit: nil,
+        # Sort order for evals by timestamp. Use `asc` for ascending order or `desc` for
+        # descending order.
+        order: nil,
+        # Evals can be ordered by creation time or last updated time. Use `created_at` for
+        # creation time or `updated_at` for last updated time.
+        order_by: nil,
+        request_options: {}
+      )
+      end
+
+      sig do
+        override.returns(
+          {
+            after: String,
+            limit: Integer,
+            order: OpenAI::EvalListParams::Order::OrSymbol,
+            order_by: OpenAI::EvalListParams::OrderBy::OrSymbol,
+            request_options: OpenAI::RequestOptions
+          }
+        )
+      end
+      def to_hash
+      end
+
+      # Sort order for evals by timestamp. Use `asc` for ascending order or `desc` for
+      # descending order.
+      module Order
+        extend OpenAI::Internal::Type::Enum
+
+        TaggedSymbol =
+          T.type_alias { T.all(Symbol, OpenAI::EvalListParams::Order) }
+        OrSymbol = T.type_alias { T.any(Symbol, String) }
+
+        ASC = T.let(:asc, OpenAI::EvalListParams::Order::TaggedSymbol)
+        DESC = T.let(:desc, OpenAI::EvalListParams::Order::TaggedSymbol)
+
+        sig do
+          override.returns(
+            T::Array[OpenAI::EvalListParams::Order::TaggedSymbol]
+          )
+        end
+        def self.values
+        end
+      end
+
+      # Evals can be ordered by creation time or last updated time. Use `created_at` for
+      # creation time or `updated_at` for last updated time.
+      module OrderBy
+        extend OpenAI::Internal::Type::Enum
+
+        TaggedSymbol =
+          T.type_alias { T.all(Symbol, OpenAI::EvalListParams::OrderBy) }
+        OrSymbol = T.type_alias { T.any(Symbol, String) }
+
+        CREATED_AT =
+          T.let(:created_at, OpenAI::EvalListParams::OrderBy::TaggedSymbol)
+        UPDATED_AT =
+          T.let(:updated_at, OpenAI::EvalListParams::OrderBy::TaggedSymbol)
+
+        sig do
+          override.returns(
+            T::Array[OpenAI::EvalListParams::OrderBy::TaggedSymbol]
+          )
+        end
+        def self.values
+        end
+      end
+    end
+  end
+end