deeprails 0.8.0 → 0.9.0

data/lib/deeprails/models/evaluation.rb

@@ -1,233 +0,0 @@
-# frozen_string_literal: true
-
-module Deeprails
-  module Models
-    # @see Deeprails::Resources::Evaluate#create
-    class Evaluation < Deeprails::Internal::Type::BaseModel
-      # @!attribute eval_id
-      #   A unique evaluation ID.
-      #
-      #   @return [String]
-      required :eval_id, String
-
-      # @!attribute evaluation_status
-      #   Status of the evaluation.
-      #
-      #   @return [Symbol, Deeprails::Models::Evaluation::EvaluationStatus]
-      required :evaluation_status, enum: -> { Deeprails::Evaluation::EvaluationStatus }
-
-      # @!attribute model_input
-      #   A dictionary of inputs sent to the LLM to generate output. The dictionary must
-      #   contain at least a `user_prompt` field or a `system_prompt` field. For
-      #   ground_truth_adherence guardrail metric, `ground_truth` should be provided.
-      #
-      #   @return [Deeprails::Models::Evaluation::ModelInput]
-      required :model_input, -> { Deeprails::Evaluation::ModelInput }
-
-      # @!attribute model_output
-      #   Output generated by the LLM to be evaluated.
-      #
-      #   @return [String]
-      required :model_output, String
-
-      # @!attribute run_mode
-      #   Run mode for the evaluation. The run mode allows the user to optimize for speed,
-      #   accuracy, and cost by determining which models are used to evaluate the event.
-      #
-      #   @return [Symbol, Deeprails::Models::Evaluation::RunMode]
-      required :run_mode, enum: -> { Deeprails::Evaluation::RunMode }
-
-      # @!attribute created_at
-      #   The time the evaluation was created in UTC.
-      #
-      #   @return [Time, nil]
-      optional :created_at, Time
-
-      # @!attribute end_timestamp
-      #   The time the evaluation completed in UTC.
-      #
-      #   @return [Time, nil]
-      optional :end_timestamp, Time
-
-      # @!attribute error_message
-      #   Description of the error causing the evaluation to fail, if any.
-      #
-      #   @return [String, nil]
-      optional :error_message, String
-
-      # @!attribute error_timestamp
-      #   The time the error causing the evaluation to fail was recorded.
-      #
-      #   @return [Time, nil]
-      optional :error_timestamp, Time
-
-      # @!attribute evaluation_result
-      #   Evaluation result consisting of average scores and rationales for each of the
-      #   evaluated guardrail metrics.
-      #
-      #   @return [Hash{Symbol=>Object}, nil]
-      optional :evaluation_result, Deeprails::Internal::Type::HashOf[Deeprails::Internal::Type::Unknown]
-
-      # @!attribute evaluation_total_cost
-      #   Total cost of the evaluation.
-      #
-      #   @return [Float, nil]
-      optional :evaluation_total_cost, Float
-
-      # @!attribute guardrail_metrics
-      #   An array of guardrail metrics that the model input and output pair will be
-      #   evaluated on.
-      #
-      #   @return [Array<Symbol, Deeprails::Models::Evaluation::GuardrailMetric>, nil]
-      optional :guardrail_metrics,
-               -> { Deeprails::Internal::Type::ArrayOf[enum: Deeprails::Evaluation::GuardrailMetric] }
-
-      # @!attribute model_used
-      #   Model ID used to generate the output, like `gpt-4o` or `o3`.
-      #
-      #   @return [String, nil]
-      optional :model_used, String
-
-      # @!attribute modified_at
-      #   The most recent time the evaluation was modified in UTC.
-      #
-      #   @return [Time, nil]
-      optional :modified_at, Time
-
-      # @!attribute nametag
-      #   An optional, user-defined tag for the evaluation.
-      #
-      #   @return [String, nil]
-      optional :nametag, String
-
-      # @!attribute progress
-      #   Evaluation progress. Values range between 0 and 100; 100 corresponds to a
-      #   completed `evaluation_status`.
-      #
-      #   @return [Integer, nil]
-      optional :progress, Integer
-
-      # @!attribute start_timestamp
-      #   The time the evaluation started in UTC.
-      #
-      #   @return [Time, nil]
-      optional :start_timestamp, Time
-
-      # @!method initialize(eval_id:, evaluation_status:, model_input:, model_output:, run_mode:, created_at: nil, end_timestamp: nil, error_message: nil, error_timestamp: nil, evaluation_result: nil, evaluation_total_cost: nil, guardrail_metrics: nil, model_used: nil, modified_at: nil, nametag: nil, progress: nil, start_timestamp: nil)
-      #   Some parameter documentations has been truncated, see
-      #   {Deeprails::Models::Evaluation} for more details.
-      #
-      #   @param eval_id [String] A unique evaluation ID.
-      #
-      #   @param evaluation_status [Symbol, Deeprails::Models::Evaluation::EvaluationStatus] Status of the evaluation.
-      #
-      #   @param model_input [Deeprails::Models::Evaluation::ModelInput] A dictionary of inputs sent to the LLM to generate output. The dictionary must c
-      #
-      #   @param model_output [String] Output generated by the LLM to be evaluated.
-      #
-      #   @param run_mode [Symbol, Deeprails::Models::Evaluation::RunMode] Run mode for the evaluation. The run mode allows the user to optimize for speed
-      #
-      #   @param created_at [Time] The time the evaluation was created in UTC.
-      #
-      #   @param end_timestamp [Time] The time the evaluation completed in UTC.
-      #
-      #   @param error_message [String] Description of the error causing the evaluation to fail, if any.
-      #
-      #   @param error_timestamp [Time] The time the error causing the evaluation to fail was recorded.
-      #
-      #   @param evaluation_result [Hash{Symbol=>Object}] Evaluation result consisting of average scores and rationales for each of the ev
-      #
-      #   @param evaluation_total_cost [Float] Total cost of the evaluation.
-      #
-      #   @param guardrail_metrics [Array<Symbol, Deeprails::Models::Evaluation::GuardrailMetric>] An array of guardrail metrics that the model input and output pair will be evalu
-      #
-      #   @param model_used [String] Model ID used to generate the output, like `gpt-4o` or `o3`.
-      #
-      #   @param modified_at [Time] The most recent time the evaluation was modified in UTC.
-      #
-      #   @param nametag [String] An optional, user-defined tag for the evaluation.
-      #
-      #   @param progress [Integer] Evaluation progress. Values range between 0 and 100; 100 corresponds to a compl
-      #
-      #   @param start_timestamp [Time] The time the evaluation started in UTC.
-
-      # Status of the evaluation.
-      #
-      # @see Deeprails::Models::Evaluation#evaluation_status
-      module EvaluationStatus
-        extend Deeprails::Internal::Type::Enum
-
-        IN_PROGRESS = :in_progress
-        COMPLETED = :completed
-        CANCELED = :canceled
-        QUEUED = :queued
-        FAILED = :failed
-
-        # @!method self.values
-        #   @return [Array<Symbol>]
-      end
-
-      # @see Deeprails::Models::Evaluation#model_input
-      class ModelInput < Deeprails::Internal::Type::BaseModel
-        # @!attribute ground_truth
-        #   The ground truth for evaluating Ground Truth Adherence guardrail.
-        #
-        #   @return [String, nil]
-        optional :ground_truth, String
-
-        # @!attribute system_prompt
-        #   The system prompt used to generate the output.
-        #
-        #   @return [String, nil]
-        optional :system_prompt, String
-
-        # @!attribute user_prompt
-        #   The user prompt used to generate the output.
-        #
-        #   @return [String, nil]
-        optional :user_prompt, String
-
-        # @!method initialize(ground_truth: nil, system_prompt: nil, user_prompt: nil)
-        #   A dictionary of inputs sent to the LLM to generate output. The dictionary must
-        #   contain at least a `user_prompt` field or a `system_prompt` field. For
-        #   ground_truth_adherence guardrail metric, `ground_truth` should be provided.
-        #
-        #   @param ground_truth [String] The ground truth for evaluating Ground Truth Adherence guardrail.
-        #
-        #   @param system_prompt [String] The system prompt used to generate the output.
-        #
-        #   @param user_prompt [String] The user prompt used to generate the output.
-      end
-
-      # Run mode for the evaluation. The run mode allows the user to optimize for speed,
-      # accuracy, and cost by determining which models are used to evaluate the event.
-      #
-      # @see Deeprails::Models::Evaluation#run_mode
-      module RunMode
-        extend Deeprails::Internal::Type::Enum
-
-        PRECISION_PLUS = :precision_plus
-        PRECISION = :precision
-        SMART = :smart
-        ECONOMY = :economy
-
-        # @!method self.values
-        #   @return [Array<Symbol>]
-      end
-
-      module GuardrailMetric
-        extend Deeprails::Internal::Type::Enum
-
-        CORRECTNESS = :correctness
-        COMPLETENESS = :completeness
-        INSTRUCTION_ADHERENCE = :instruction_adherence
-        CONTEXT_ADHERENCE = :context_adherence
-        GROUND_TRUTH_ADHERENCE = :ground_truth_adherence
-        COMPREHENSIVE_SAFETY = :comprehensive_safety
-
-        # @!method self.values
-        #   @return [Array<Symbol>]
-      end
-    end
-  end
-end

data/lib/deeprails/resources/evaluate.rb

@@ -1,70 +0,0 @@
-# frozen_string_literal: true
-
-module Deeprails
-  module Resources
-    class Evaluate
-      # Some parameter documentations has been truncated, see
-      # {Deeprails::Models::EvaluateCreateParams} for more details.
-      #
-      # Use this endpoint to evaluate a model's input and output pair against selected
-      # guardrail metrics
-      #
-      # @overload create(model_input:, model_output:, run_mode:, guardrail_metrics: nil, model_used: nil, nametag: nil, request_options: {})
-      #
-      # @param model_input [Deeprails::Models::EvaluateCreateParams::ModelInput] A dictionary of inputs sent to the LLM to generate output. The dictionary must c
-      #
-      # @param model_output [String] Output generated by the LLM to be evaluated.
-      #
-      # @param run_mode [Symbol, Deeprails::Models::EvaluateCreateParams::RunMode] Run mode for the evaluation. The run mode allows the user to optimize for speed
-      #
-      # @param guardrail_metrics [Array<Symbol, Deeprails::Models::EvaluateCreateParams::GuardrailMetric>] An array of guardrail metrics that the model input and output pair will be evalu
-      #
-      # @param model_used [String] Model ID used to generate the output, like `gpt-4o` or `o3`.
-      #
-      # @param nametag [String] An optional, user-defined tag for the evaluation.
-      #
-      # @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}, nil]
-      #
-      # @return [Deeprails::Models::Evaluation]
-      #
-      # @see Deeprails::Models::EvaluateCreateParams
-      def create(params)
-        parsed, options = Deeprails::EvaluateCreateParams.dump_request(params)
-        @client.request(
-          method: :post,
-          path: "evaluate",
-          body: parsed,
-          model: Deeprails::Evaluation,
-          options: options
-        )
-      end
-
-      # Use this endpoint to retrieve the evaluation record for a given evaluation ID
-      #
-      # @overload retrieve(eval_id, request_options: {})
-      #
-      # @param eval_id [String] The ID of the evaluation to retrieve.
-      #
-      # @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}, nil]
-      #
-      # @return [Deeprails::Models::Evaluation]
-      #
-      # @see Deeprails::Models::EvaluateRetrieveParams
-      def retrieve(eval_id, params = {})
-        @client.request(
-          method: :get,
-          path: ["evaluate/%1$s", eval_id],
-          model: Deeprails::Evaluation,
-          options: params[:request_options]
-        )
-      end
-
-      # @api private
-      #
-      # @param client [Deeprails::Client]
-      def initialize(client:)
-        @client = client
-      end
-    end
-  end
-end

data/rbi/deeprails/models/evaluate_create_params.rbi

@@ -1,280 +0,0 @@
-# typed: strong
-
-module Deeprails
-  module Models
-    class EvaluateCreateParams < Deeprails::Internal::Type::BaseModel
-      extend Deeprails::Internal::Type::RequestParameters::Converter
-      include Deeprails::Internal::Type::RequestParameters
-
-      OrHash =
-        T.type_alias do
-          T.any(Deeprails::EvaluateCreateParams, Deeprails::Internal::AnyHash)
-        end
-
-      # A dictionary of inputs sent to the LLM to generate output. The dictionary must
-      # contain at least a `user_prompt` field or a `system_prompt` field. For
-      # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
-      sig { returns(Deeprails::EvaluateCreateParams::ModelInput) }
-      attr_reader :model_input
-
-      sig do
-        params(
-          model_input: Deeprails::EvaluateCreateParams::ModelInput::OrHash
-        ).void
-      end
-      attr_writer :model_input
-
-      # Output generated by the LLM to be evaluated.
-      sig { returns(String) }
-      attr_accessor :model_output
-
-      # Run mode for the evaluation. The run mode allows the user to optimize for speed,
-      # accuracy, and cost by determining which models are used to evaluate the event.
-      # Available run modes include `precision_plus`, `precision`, `smart`, and
-      # `economy`. Defaults to `smart`.
-      sig { returns(Deeprails::EvaluateCreateParams::RunMode::OrSymbol) }
-      attr_accessor :run_mode
-
-      # An array of guardrail metrics that the model input and output pair will be
-      # evaluated on. For non-enterprise users, these will be limited to the allowed
-      # guardrail metrics.
-      sig do
-        returns(
-          T.nilable(
-            T::Array[Deeprails::EvaluateCreateParams::GuardrailMetric::OrSymbol]
-          )
-        )
-      end
-      attr_reader :guardrail_metrics
-
-      sig do
-        params(
-          guardrail_metrics:
-            T::Array[Deeprails::EvaluateCreateParams::GuardrailMetric::OrSymbol]
-        ).void
-      end
-      attr_writer :guardrail_metrics
-
-      # Model ID used to generate the output, like `gpt-4o` or `o3`.
-      sig { returns(T.nilable(String)) }
-      attr_reader :model_used
-
-      sig { params(model_used: String).void }
-      attr_writer :model_used
-
-      # An optional, user-defined tag for the evaluation.
-      sig { returns(T.nilable(String)) }
-      attr_reader :nametag
-
-      sig { params(nametag: String).void }
-      attr_writer :nametag
-
-      sig do
-        params(
-          model_input: Deeprails::EvaluateCreateParams::ModelInput::OrHash,
-          model_output: String,
-          run_mode: Deeprails::EvaluateCreateParams::RunMode::OrSymbol,
-          guardrail_metrics:
-            T::Array[
-              Deeprails::EvaluateCreateParams::GuardrailMetric::OrSymbol
-            ],
-          model_used: String,
-          nametag: String,
-          request_options: Deeprails::RequestOptions::OrHash
-        ).returns(T.attached_class)
-      end
-      def self.new(
-        # A dictionary of inputs sent to the LLM to generate output. The dictionary must
-        # contain at least a `user_prompt` field or a `system_prompt` field. For
-        # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
-        model_input:,
-        # Output generated by the LLM to be evaluated.
-        model_output:,
-        # Run mode for the evaluation. The run mode allows the user to optimize for speed,
-        # accuracy, and cost by determining which models are used to evaluate the event.
-        # Available run modes include `precision_plus`, `precision`, `smart`, and
-        # `economy`. Defaults to `smart`.
-        run_mode:,
-        # An array of guardrail metrics that the model input and output pair will be
-        # evaluated on. For non-enterprise users, these will be limited to the allowed
-        # guardrail metrics.
-        guardrail_metrics: nil,
-        # Model ID used to generate the output, like `gpt-4o` or `o3`.
-        model_used: nil,
-        # An optional, user-defined tag for the evaluation.
-        nametag: nil,
-        request_options: {}
-      )
-      end
-
-      sig do
-        override.returns(
-          {
-            model_input: Deeprails::EvaluateCreateParams::ModelInput,
-            model_output: String,
-            run_mode: Deeprails::EvaluateCreateParams::RunMode::OrSymbol,
-            guardrail_metrics:
-              T::Array[
-                Deeprails::EvaluateCreateParams::GuardrailMetric::OrSymbol
-              ],
-            model_used: String,
-            nametag: String,
-            request_options: Deeprails::RequestOptions
-          }
-        )
-      end
-      def to_hash
-      end
-
-      class ModelInput < Deeprails::Internal::Type::BaseModel
-        OrHash =
-          T.type_alias do
-            T.any(
-              Deeprails::EvaluateCreateParams::ModelInput,
-              Deeprails::Internal::AnyHash
-            )
-          end
-
-        # The ground truth for evaluating Ground Truth Adherence guardrail.
-        sig { returns(T.nilable(String)) }
-        attr_reader :ground_truth
-
-        sig { params(ground_truth: String).void }
-        attr_writer :ground_truth
-
-        # The system prompt used to generate the output.
-        sig { returns(T.nilable(String)) }
-        attr_reader :system_prompt
-
-        sig { params(system_prompt: String).void }
-        attr_writer :system_prompt
-
-        # The user prompt used to generate the output.
-        sig { returns(T.nilable(String)) }
-        attr_reader :user_prompt
-
-        sig { params(user_prompt: String).void }
-        attr_writer :user_prompt
-
-        # A dictionary of inputs sent to the LLM to generate output. The dictionary must
-        # contain at least a `user_prompt` field or a `system_prompt` field. For
-        # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
-        sig do
-          params(
-            ground_truth: String,
-            system_prompt: String,
-            user_prompt: String
-          ).returns(T.attached_class)
-        end
-        def self.new(
-          # The ground truth for evaluating Ground Truth Adherence guardrail.
-          ground_truth: nil,
-          # The system prompt used to generate the output.
-          system_prompt: nil,
-          # The user prompt used to generate the output.
-          user_prompt: nil
-        )
-        end
-
-        sig do
-          override.returns(
-            { ground_truth: String, system_prompt: String, user_prompt: String }
-          )
-        end
-        def to_hash
-        end
-      end
-
-      # Run mode for the evaluation. The run mode allows the user to optimize for speed,
-      # accuracy, and cost by determining which models are used to evaluate the event.
-      # Available run modes include `precision_plus`, `precision`, `smart`, and
-      # `economy`. Defaults to `smart`.
-      module RunMode
-        extend Deeprails::Internal::Type::Enum
-
-        TaggedSymbol =
-          T.type_alias do
-            T.all(Symbol, Deeprails::EvaluateCreateParams::RunMode)
-          end
-        OrSymbol = T.type_alias { T.any(Symbol, String) }
-
-        PRECISION_PLUS =
-          T.let(
-            :precision_plus,
-            Deeprails::EvaluateCreateParams::RunMode::TaggedSymbol
-          )
-        PRECISION =
-          T.let(
-            :precision,
-            Deeprails::EvaluateCreateParams::RunMode::TaggedSymbol
-          )
-        SMART =
-          T.let(:smart, Deeprails::EvaluateCreateParams::RunMode::TaggedSymbol)
-        ECONOMY =
-          T.let(
-            :economy,
-            Deeprails::EvaluateCreateParams::RunMode::TaggedSymbol
-          )
-
-        sig do
-          override.returns(
-            T::Array[Deeprails::EvaluateCreateParams::RunMode::TaggedSymbol]
-          )
-        end
-        def self.values
-        end
-      end
-
-      module GuardrailMetric
-        extend Deeprails::Internal::Type::Enum
-
-        TaggedSymbol =
-          T.type_alias do
-            T.all(Symbol, Deeprails::EvaluateCreateParams::GuardrailMetric)
-          end
-        OrSymbol = T.type_alias { T.any(Symbol, String) }
-
-        CORRECTNESS =
-          T.let(
-            :correctness,
-            Deeprails::EvaluateCreateParams::GuardrailMetric::TaggedSymbol
-          )
-        COMPLETENESS =
-          T.let(
-            :completeness,
-            Deeprails::EvaluateCreateParams::GuardrailMetric::TaggedSymbol
-          )
-        INSTRUCTION_ADHERENCE =
-          T.let(
-            :instruction_adherence,
-            Deeprails::EvaluateCreateParams::GuardrailMetric::TaggedSymbol
-          )
-        CONTEXT_ADHERENCE =
-          T.let(
-            :context_adherence,
-            Deeprails::EvaluateCreateParams::GuardrailMetric::TaggedSymbol
-          )
-        GROUND_TRUTH_ADHERENCE =
-          T.let(
-            :ground_truth_adherence,
-            Deeprails::EvaluateCreateParams::GuardrailMetric::TaggedSymbol
-          )
-        COMPREHENSIVE_SAFETY =
-          T.let(
-            :comprehensive_safety,
-            Deeprails::EvaluateCreateParams::GuardrailMetric::TaggedSymbol
-          )
-
-        sig do
-          override.returns(
-            T::Array[
-              Deeprails::EvaluateCreateParams::GuardrailMetric::TaggedSymbol
-            ]
-          )
-        end
-        def self.values
-        end
-      end
-    end
-  end
-end

data/rbi/deeprails/models/evaluate_retrieve_params.rbi

@@ -1,27 +0,0 @@
-# typed: strong
-
-module Deeprails
-  module Models
-    class EvaluateRetrieveParams < Deeprails::Internal::Type::BaseModel
-      extend Deeprails::Internal::Type::RequestParameters::Converter
-      include Deeprails::Internal::Type::RequestParameters
-
-      OrHash =
-        T.type_alias do
-          T.any(Deeprails::EvaluateRetrieveParams, Deeprails::Internal::AnyHash)
-        end
-
-      sig do
-        params(request_options: Deeprails::RequestOptions::OrHash).returns(
-          T.attached_class
-        )
-      end
-      def self.new(request_options: {})
-      end
-
-      sig { override.returns({ request_options: Deeprails::RequestOptions }) }
-      def to_hash
-      end
-    end
-  end
-end