deeprails 0.7.0 → 0.9.0

data/rbi/deeprails/models/evaluation.rbi

@@ -1,402 +0,0 @@
-# typed: strong
-
-module Deeprails
-  module Models
-    class Evaluation < Deeprails::Internal::Type::BaseModel
-      OrHash =
-        T.type_alias do
-          T.any(Deeprails::Evaluation, Deeprails::Internal::AnyHash)
-        end
-
-      # A unique evaluation ID.
-      sig { returns(String) }
-      attr_accessor :eval_id
-
-      # Status of the evaluation.
-      sig { returns(Deeprails::Evaluation::EvaluationStatus::TaggedSymbol) }
-      attr_accessor :evaluation_status
-
-      # A dictionary of inputs sent to the LLM to generate output. The dictionary must
-      # contain at least `user_prompt` or `system_prompt` field. For
-      # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
-      sig { returns(Deeprails::Evaluation::ModelInput) }
-      attr_reader :model_input
-
-      sig do
-        params(model_input: Deeprails::Evaluation::ModelInput::OrHash).void
-      end
-      attr_writer :model_input
-
-      # Output generated by the LLM to be evaluated.
-      sig { returns(String) }
-      attr_accessor :model_output
-
-      # Run mode for the evaluation. The run mode allows the user to optimize for speed,
-      # accuracy, and cost by determining which models are used to evaluate the event.
-      sig { returns(Deeprails::Evaluation::RunMode::TaggedSymbol) }
-      attr_accessor :run_mode
-
-      # The time the evaluation was created in UTC.
-      sig { returns(T.nilable(Time)) }
-      attr_reader :created_at
-
-      sig { params(created_at: Time).void }
-      attr_writer :created_at
-
-      # The time the evaluation completed in UTC.
-      sig { returns(T.nilable(Time)) }
-      attr_reader :end_timestamp
-
-      sig { params(end_timestamp: Time).void }
-      attr_writer :end_timestamp
-
-      # Description of the error causing the evaluation to fail, if any.
-      sig { returns(T.nilable(String)) }
-      attr_reader :error_message
-
-      sig { params(error_message: String).void }
-      attr_writer :error_message
-
-      # The time the error causing the evaluation to fail was recorded.
-      sig { returns(T.nilable(Time)) }
-      attr_reader :error_timestamp
-
-      sig { params(error_timestamp: Time).void }
-      attr_writer :error_timestamp
-
-      # Evaluation result consisting of average scores and rationales for each of the
-      # evaluated guardrail metrics.
-      sig { returns(T.nilable(T::Hash[Symbol, T.anything])) }
-      attr_reader :evaluation_result
-
-      sig { params(evaluation_result: T::Hash[Symbol, T.anything]).void }
-      attr_writer :evaluation_result
-
-      # Total cost of the evaluation.
-      sig { returns(T.nilable(Float)) }
-      attr_reader :evaluation_total_cost
-
-      sig { params(evaluation_total_cost: Float).void }
-      attr_writer :evaluation_total_cost
-
-      # An array of guardrail metrics that the model input and output pair will be
-      # evaluated on.
-      sig do
-        returns(
-          T.nilable(
-            T::Array[Deeprails::Evaluation::GuardrailMetric::TaggedSymbol]
-          )
-        )
-      end
-      attr_reader :guardrail_metrics
-
-      sig do
-        params(
-          guardrail_metrics:
-            T::Array[Deeprails::Evaluation::GuardrailMetric::OrSymbol]
-        ).void
-      end
-      attr_writer :guardrail_metrics
-
-      # Model ID used to generate the output, like `gpt-4o` or `o3`.
-      sig { returns(T.nilable(String)) }
-      attr_reader :model_used
-
-      sig { params(model_used: String).void }
-      attr_writer :model_used
-
-      # The most recent time the evaluation was modified in UTC.
-      sig { returns(T.nilable(Time)) }
-      attr_reader :modified_at
-
-      sig { params(modified_at: Time).void }
-      attr_writer :modified_at
-
-      # An optional, user-defined tag for the evaluation.
-      sig { returns(T.nilable(String)) }
-      attr_reader :nametag
-
-      sig { params(nametag: String).void }
-      attr_writer :nametag
-
-      # Evaluation progress. Values range between 0 and 100; 100 corresponds to a
-      # completed `evaluation_status`.
-      sig { returns(T.nilable(Integer)) }
-      attr_reader :progress
-
-      sig { params(progress: Integer).void }
-      attr_writer :progress
-
-      # The time the evaluation started in UTC.
-      sig { returns(T.nilable(Time)) }
-      attr_reader :start_timestamp
-
-      sig { params(start_timestamp: Time).void }
-      attr_writer :start_timestamp
-
-      sig do
-        params(
-          eval_id: String,
-          evaluation_status: Deeprails::Evaluation::EvaluationStatus::OrSymbol,
-          model_input: Deeprails::Evaluation::ModelInput::OrHash,
-          model_output: String,
-          run_mode: Deeprails::Evaluation::RunMode::OrSymbol,
-          created_at: Time,
-          end_timestamp: Time,
-          error_message: String,
-          error_timestamp: Time,
-          evaluation_result: T::Hash[Symbol, T.anything],
-          evaluation_total_cost: Float,
-          guardrail_metrics:
-            T::Array[Deeprails::Evaluation::GuardrailMetric::OrSymbol],
-          model_used: String,
-          modified_at: Time,
-          nametag: String,
-          progress: Integer,
-          start_timestamp: Time
-        ).returns(T.attached_class)
-      end
-      def self.new(
-        # A unique evaluation ID.
-        eval_id:,
-        # Status of the evaluation.
-        evaluation_status:,
-        # A dictionary of inputs sent to the LLM to generate output. The dictionary must
-        # contain at least `user_prompt` or `system_prompt` field. For
-        # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
-        model_input:,
-        # Output generated by the LLM to be evaluated.
-        model_output:,
-        # Run mode for the evaluation. The run mode allows the user to optimize for speed,
-        # accuracy, and cost by determining which models are used to evaluate the event.
-        run_mode:,
-        # The time the evaluation was created in UTC.
-        created_at: nil,
-        # The time the evaluation completed in UTC.
-        end_timestamp: nil,
-        # Description of the error causing the evaluation to fail, if any.
-        error_message: nil,
-        # The time the error causing the evaluation to fail was recorded.
-        error_timestamp: nil,
-        # Evaluation result consisting of average scores and rationales for each of the
-        # evaluated guardrail metrics.
-        evaluation_result: nil,
-        # Total cost of the evaluation.
-        evaluation_total_cost: nil,
-        # An array of guardrail metrics that the model input and output pair will be
-        # evaluated on.
-        guardrail_metrics: nil,
-        # Model ID used to generate the output, like `gpt-4o` or `o3`.
-        model_used: nil,
-        # The most recent time the evaluation was modified in UTC.
-        modified_at: nil,
-        # An optional, user-defined tag for the evaluation.
-        nametag: nil,
-        # Evaluation progress. Values range between 0 and 100; 100 corresponds to a
-        # completed `evaluation_status`.
-        progress: nil,
-        # The time the evaluation started in UTC.
-        start_timestamp: nil
-      )
-      end
-
-      sig do
-        override.returns(
-          {
-            eval_id: String,
-            evaluation_status:
-              Deeprails::Evaluation::EvaluationStatus::TaggedSymbol,
-            model_input: Deeprails::Evaluation::ModelInput,
-            model_output: String,
-            run_mode: Deeprails::Evaluation::RunMode::TaggedSymbol,
-            created_at: Time,
-            end_timestamp: Time,
-            error_message: String,
-            error_timestamp: Time,
-            evaluation_result: T::Hash[Symbol, T.anything],
-            evaluation_total_cost: Float,
-            guardrail_metrics:
-              T::Array[Deeprails::Evaluation::GuardrailMetric::TaggedSymbol],
-            model_used: String,
-            modified_at: Time,
-            nametag: String,
-            progress: Integer,
-            start_timestamp: Time
-          }
-        )
-      end
-      def to_hash
-      end
-
-      # Status of the evaluation.
-      module EvaluationStatus
-        extend Deeprails::Internal::Type::Enum
-
-        TaggedSymbol =
-          T.type_alias do
-            T.all(Symbol, Deeprails::Evaluation::EvaluationStatus)
-          end
-        OrSymbol = T.type_alias { T.any(Symbol, String) }
-
-        IN_PROGRESS =
-          T.let(
-            :in_progress,
-            Deeprails::Evaluation::EvaluationStatus::TaggedSymbol
-          )
-        COMPLETED =
-          T.let(
-            :completed,
-            Deeprails::Evaluation::EvaluationStatus::TaggedSymbol
-          )
-        CANCELED =
-          T.let(
-            :canceled,
-            Deeprails::Evaluation::EvaluationStatus::TaggedSymbol
-          )
-        QUEUED =
-          T.let(:queued, Deeprails::Evaluation::EvaluationStatus::TaggedSymbol)
-        FAILED =
-          T.let(:failed, Deeprails::Evaluation::EvaluationStatus::TaggedSymbol)
-
-        sig do
-          override.returns(
-            T::Array[Deeprails::Evaluation::EvaluationStatus::TaggedSymbol]
-          )
-        end
-        def self.values
-        end
-      end
-
-      class ModelInput < Deeprails::Internal::Type::BaseModel
-        OrHash =
-          T.type_alias do
-            T.any(
-              Deeprails::Evaluation::ModelInput,
-              Deeprails::Internal::AnyHash
-            )
-          end
-
-        # The ground truth for evaluating Ground Truth Adherence guardrail.
-        sig { returns(T.nilable(String)) }
-        attr_reader :ground_truth
-
-        sig { params(ground_truth: String).void }
-        attr_writer :ground_truth
-
-        # The system prompt used to generate the output.
-        sig { returns(T.nilable(String)) }
-        attr_reader :system_prompt
-
-        sig { params(system_prompt: String).void }
-        attr_writer :system_prompt
-
-        # The user prompt used to generate the output.
-        sig { returns(T.nilable(String)) }
-        attr_reader :user_prompt
-
-        sig { params(user_prompt: String).void }
-        attr_writer :user_prompt
-
-        # A dictionary of inputs sent to the LLM to generate output. The dictionary must
-        # contain at least `user_prompt` or `system_prompt` field. For
-        # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
-        sig do
-          params(
-            ground_truth: String,
-            system_prompt: String,
-            user_prompt: String
-          ).returns(T.attached_class)
-        end
-        def self.new(
-          # The ground truth for evaluating Ground Truth Adherence guardrail.
-          ground_truth: nil,
-          # The system prompt used to generate the output.
-          system_prompt: nil,
-          # The user prompt used to generate the output.
-          user_prompt: nil
-        )
-        end
-
-        sig do
-          override.returns(
-            { ground_truth: String, system_prompt: String, user_prompt: String }
-          )
-        end
-        def to_hash
-        end
-      end
-
-      # Run mode for the evaluation. The run mode allows the user to optimize for speed,
-      # accuracy, and cost by determining which models are used to evaluate the event.
-      module RunMode
-        extend Deeprails::Internal::Type::Enum
-
-        TaggedSymbol =
-          T.type_alias { T.all(Symbol, Deeprails::Evaluation::RunMode) }
-        OrSymbol = T.type_alias { T.any(Symbol, String) }
-
-        PRECISION_PLUS =
-          T.let(:precision_plus, Deeprails::Evaluation::RunMode::TaggedSymbol)
-        PRECISION =
-          T.let(:precision, Deeprails::Evaluation::RunMode::TaggedSymbol)
-        SMART = T.let(:smart, Deeprails::Evaluation::RunMode::TaggedSymbol)
-        ECONOMY = T.let(:economy, Deeprails::Evaluation::RunMode::TaggedSymbol)
-
-        sig do
-          override.returns(
-            T::Array[Deeprails::Evaluation::RunMode::TaggedSymbol]
-          )
-        end
-        def self.values
-        end
-      end
-
-      module GuardrailMetric
-        extend Deeprails::Internal::Type::Enum
-
-        TaggedSymbol =
-          T.type_alias { T.all(Symbol, Deeprails::Evaluation::GuardrailMetric) }
-        OrSymbol = T.type_alias { T.any(Symbol, String) }
-
-        CORRECTNESS =
-          T.let(
-            :correctness,
-            Deeprails::Evaluation::GuardrailMetric::TaggedSymbol
-          )
-        COMPLETENESS =
-          T.let(
-            :completeness,
-            Deeprails::Evaluation::GuardrailMetric::TaggedSymbol
-          )
-        INSTRUCTION_ADHERENCE =
-          T.let(
-            :instruction_adherence,
-            Deeprails::Evaluation::GuardrailMetric::TaggedSymbol
-          )
-        CONTEXT_ADHERENCE =
-          T.let(
-            :context_adherence,
-            Deeprails::Evaluation::GuardrailMetric::TaggedSymbol
-          )
-        GROUND_TRUTH_ADHERENCE =
-          T.let(
-            :ground_truth_adherence,
-            Deeprails::Evaluation::GuardrailMetric::TaggedSymbol
-          )
-        COMPREHENSIVE_SAFETY =
-          T.let(
-            :comprehensive_safety,
-            Deeprails::Evaluation::GuardrailMetric::TaggedSymbol
-          )
-
-        sig do
-          override.returns(
-            T::Array[Deeprails::Evaluation::GuardrailMetric::TaggedSymbol]
-          )
-        end
-        def self.values
-        end
-      end
-    end
-  end
-end