deeprails 0.8.0 → 0.9.0

data/rbi/deeprails/models/evaluation.rbi

@@ -1,402 +0,0 @@
-# typed: strong
-
-module Deeprails
-  module Models
-    class Evaluation < Deeprails::Internal::Type::BaseModel
-      OrHash =
-        T.type_alias do
-          T.any(Deeprails::Evaluation, Deeprails::Internal::AnyHash)
-        end
-
-      # A unique evaluation ID.
-      sig { returns(String) }
-      attr_accessor :eval_id
-
-      # Status of the evaluation.
-      sig { returns(Deeprails::Evaluation::EvaluationStatus::TaggedSymbol) }
-      attr_accessor :evaluation_status
-
-      # A dictionary of inputs sent to the LLM to generate output. The dictionary must
-      # contain at least a `user_prompt` field or a `system_prompt` field. For
-      # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
-      sig { returns(Deeprails::Evaluation::ModelInput) }
-      attr_reader :model_input
-
-      sig do
-        params(model_input: Deeprails::Evaluation::ModelInput::OrHash).void
-      end
-      attr_writer :model_input
-
-      # Output generated by the LLM to be evaluated.
-      sig { returns(String) }
-      attr_accessor :model_output
-
-      # Run mode for the evaluation. The run mode allows the user to optimize for speed,
-      # accuracy, and cost by determining which models are used to evaluate the event.
-      sig { returns(Deeprails::Evaluation::RunMode::TaggedSymbol) }
-      attr_accessor :run_mode
-
-      # The time the evaluation was created in UTC.
-      sig { returns(T.nilable(Time)) }
-      attr_reader :created_at
-
-      sig { params(created_at: Time).void }
-      attr_writer :created_at
-
-      # The time the evaluation completed in UTC.
-      sig { returns(T.nilable(Time)) }
-      attr_reader :end_timestamp
-
-      sig { params(end_timestamp: Time).void }
-      attr_writer :end_timestamp
-
-      # Description of the error causing the evaluation to fail, if any.
-      sig { returns(T.nilable(String)) }
-      attr_reader :error_message
-
-      sig { params(error_message: String).void }
-      attr_writer :error_message
-
-      # The time the error causing the evaluation to fail was recorded.
-      sig { returns(T.nilable(Time)) }
-      attr_reader :error_timestamp
-
-      sig { params(error_timestamp: Time).void }
-      attr_writer :error_timestamp
-
-      # Evaluation result consisting of average scores and rationales for each of the
-      # evaluated guardrail metrics.
-      sig { returns(T.nilable(T::Hash[Symbol, T.anything])) }
-      attr_reader :evaluation_result
-
-      sig { params(evaluation_result: T::Hash[Symbol, T.anything]).void }
-      attr_writer :evaluation_result
-
-      # Total cost of the evaluation.
-      sig { returns(T.nilable(Float)) }
-      attr_reader :evaluation_total_cost
-
-      sig { params(evaluation_total_cost: Float).void }
-      attr_writer :evaluation_total_cost
-
-      # An array of guardrail metrics that the model input and output pair will be
-      # evaluated on.
-      sig do
-        returns(
-          T.nilable(
-            T::Array[Deeprails::Evaluation::GuardrailMetric::TaggedSymbol]
-          )
-        )
-      end
-      attr_reader :guardrail_metrics
-
-      sig do
-        params(
-          guardrail_metrics:
-            T::Array[Deeprails::Evaluation::GuardrailMetric::OrSymbol]
-        ).void
-      end
-      attr_writer :guardrail_metrics
-
-      # Model ID used to generate the output, like `gpt-4o` or `o3`.
-      sig { returns(T.nilable(String)) }
-      attr_reader :model_used
-
-      sig { params(model_used: String).void }
-      attr_writer :model_used
-
-      # The most recent time the evaluation was modified in UTC.
-      sig { returns(T.nilable(Time)) }
-      attr_reader :modified_at
-
-      sig { params(modified_at: Time).void }
-      attr_writer :modified_at
-
-      # An optional, user-defined tag for the evaluation.
-      sig { returns(T.nilable(String)) }
-      attr_reader :nametag
-
-      sig { params(nametag: String).void }
-      attr_writer :nametag
-
-      # Evaluation progress. Values range between 0 and 100; 100 corresponds to a
-      # completed `evaluation_status`.
-      sig { returns(T.nilable(Integer)) }
-      attr_reader :progress
-
-      sig { params(progress: Integer).void }
-      attr_writer :progress
-
-      # The time the evaluation started in UTC.
-      sig { returns(T.nilable(Time)) }
-      attr_reader :start_timestamp
-
-      sig { params(start_timestamp: Time).void }
-      attr_writer :start_timestamp
-
-      sig do
-        params(
-          eval_id: String,
-          evaluation_status: Deeprails::Evaluation::EvaluationStatus::OrSymbol,
-          model_input: Deeprails::Evaluation::ModelInput::OrHash,
-          model_output: String,
-          run_mode: Deeprails::Evaluation::RunMode::OrSymbol,
-          created_at: Time,
-          end_timestamp: Time,
-          error_message: String,
-          error_timestamp: Time,
-          evaluation_result: T::Hash[Symbol, T.anything],
-          evaluation_total_cost: Float,
-          guardrail_metrics:
-            T::Array[Deeprails::Evaluation::GuardrailMetric::OrSymbol],
-          model_used: String,
-          modified_at: Time,
-          nametag: String,
-          progress: Integer,
-          start_timestamp: Time
-        ).returns(T.attached_class)
-      end
-      def self.new(
-        # A unique evaluation ID.
-        eval_id:,
-        # Status of the evaluation.
-        evaluation_status:,
-        # A dictionary of inputs sent to the LLM to generate output. The dictionary must
-        # contain at least a `user_prompt` field or a `system_prompt` field. For
-        # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
-        model_input:,
-        # Output generated by the LLM to be evaluated.
-        model_output:,
-        # Run mode for the evaluation. The run mode allows the user to optimize for speed,
-        # accuracy, and cost by determining which models are used to evaluate the event.
-        run_mode:,
-        # The time the evaluation was created in UTC.
-        created_at: nil,
-        # The time the evaluation completed in UTC.
-        end_timestamp: nil,
-        # Description of the error causing the evaluation to fail, if any.
-        error_message: nil,
-        # The time the error causing the evaluation to fail was recorded.
-        error_timestamp: nil,
-        # Evaluation result consisting of average scores and rationales for each of the
-        # evaluated guardrail metrics.
-        evaluation_result: nil,
-        # Total cost of the evaluation.
-        evaluation_total_cost: nil,
-        # An array of guardrail metrics that the model input and output pair will be
-        # evaluated on.
-        guardrail_metrics: nil,
-        # Model ID used to generate the output, like `gpt-4o` or `o3`.
-        model_used: nil,
-        # The most recent time the evaluation was modified in UTC.
-        modified_at: nil,
-        # An optional, user-defined tag for the evaluation.
-        nametag: nil,
-        # Evaluation progress. Values range between 0 and 100; 100 corresponds to a
-        # completed `evaluation_status`.
-        progress: nil,
-        # The time the evaluation started in UTC.
-        start_timestamp: nil
-      )
-      end
-
-      sig do
-        override.returns(
-          {
-            eval_id: String,
-            evaluation_status:
-              Deeprails::Evaluation::EvaluationStatus::TaggedSymbol,
-            model_input: Deeprails::Evaluation::ModelInput,
-            model_output: String,
-            run_mode: Deeprails::Evaluation::RunMode::TaggedSymbol,
-            created_at: Time,
-            end_timestamp: Time,
-            error_message: String,
-            error_timestamp: Time,
-            evaluation_result: T::Hash[Symbol, T.anything],
-            evaluation_total_cost: Float,
-            guardrail_metrics:
-              T::Array[Deeprails::Evaluation::GuardrailMetric::TaggedSymbol],
-            model_used: String,
-            modified_at: Time,
-            nametag: String,
-            progress: Integer,
-            start_timestamp: Time
-          }
-        )
-      end
-      def to_hash
-      end
-
-      # Status of the evaluation.
-      module EvaluationStatus
-        extend Deeprails::Internal::Type::Enum
-
-        TaggedSymbol =
-          T.type_alias do
-            T.all(Symbol, Deeprails::Evaluation::EvaluationStatus)
-          end
-        OrSymbol = T.type_alias { T.any(Symbol, String) }
-
-        IN_PROGRESS =
-          T.let(
-            :in_progress,
-            Deeprails::Evaluation::EvaluationStatus::TaggedSymbol
-          )
-        COMPLETED =
-          T.let(
-            :completed,
-            Deeprails::Evaluation::EvaluationStatus::TaggedSymbol
-          )
-        CANCELED =
-          T.let(
-            :canceled,
-            Deeprails::Evaluation::EvaluationStatus::TaggedSymbol
-          )
-        QUEUED =
-          T.let(:queued, Deeprails::Evaluation::EvaluationStatus::TaggedSymbol)
-        FAILED =
-          T.let(:failed, Deeprails::Evaluation::EvaluationStatus::TaggedSymbol)
-
-        sig do
-          override.returns(
-            T::Array[Deeprails::Evaluation::EvaluationStatus::TaggedSymbol]
-          )
-        end
-        def self.values
-        end
-      end
-
-      class ModelInput < Deeprails::Internal::Type::BaseModel
-        OrHash =
-          T.type_alias do
-            T.any(
-              Deeprails::Evaluation::ModelInput,
-              Deeprails::Internal::AnyHash
-            )
-          end
-
-        # The ground truth for evaluating Ground Truth Adherence guardrail.
-        sig { returns(T.nilable(String)) }
-        attr_reader :ground_truth
-
-        sig { params(ground_truth: String).void }
-        attr_writer :ground_truth
-
-        # The system prompt used to generate the output.
-        sig { returns(T.nilable(String)) }
-        attr_reader :system_prompt
-
-        sig { params(system_prompt: String).void }
-        attr_writer :system_prompt
-
-        # The user prompt used to generate the output.
-        sig { returns(T.nilable(String)) }
-        attr_reader :user_prompt
-
-        sig { params(user_prompt: String).void }
-        attr_writer :user_prompt
-
-        # A dictionary of inputs sent to the LLM to generate output. The dictionary must
-        # contain at least a `user_prompt` field or a `system_prompt` field. For
-        # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
-        sig do
-          params(
-            ground_truth: String,
-            system_prompt: String,
-            user_prompt: String
-          ).returns(T.attached_class)
-        end
-        def self.new(
-          # The ground truth for evaluating Ground Truth Adherence guardrail.
-          ground_truth: nil,
-          # The system prompt used to generate the output.
-          system_prompt: nil,
-          # The user prompt used to generate the output.
-          user_prompt: nil
-        )
-        end
-
-        sig do
-          override.returns(
-            { ground_truth: String, system_prompt: String, user_prompt: String }
-          )
-        end
-        def to_hash
-        end
-      end
-
-      # Run mode for the evaluation. The run mode allows the user to optimize for speed,
-      # accuracy, and cost by determining which models are used to evaluate the event.
-      module RunMode
-        extend Deeprails::Internal::Type::Enum
-
-        TaggedSymbol =
-          T.type_alias { T.all(Symbol, Deeprails::Evaluation::RunMode) }
-        OrSymbol = T.type_alias { T.any(Symbol, String) }
-
-        PRECISION_PLUS =
-          T.let(:precision_plus, Deeprails::Evaluation::RunMode::TaggedSymbol)
-        PRECISION =
-          T.let(:precision, Deeprails::Evaluation::RunMode::TaggedSymbol)
-        SMART = T.let(:smart, Deeprails::Evaluation::RunMode::TaggedSymbol)
-        ECONOMY = T.let(:economy, Deeprails::Evaluation::RunMode::TaggedSymbol)
-
-        sig do
-          override.returns(
-            T::Array[Deeprails::Evaluation::RunMode::TaggedSymbol]
-          )
-        end
-        def self.values
-        end
-      end
-
-      module GuardrailMetric
-        extend Deeprails::Internal::Type::Enum
-
-        TaggedSymbol =
-          T.type_alias { T.all(Symbol, Deeprails::Evaluation::GuardrailMetric) }
-        OrSymbol = T.type_alias { T.any(Symbol, String) }
-
-        CORRECTNESS =
-          T.let(
-            :correctness,
-            Deeprails::Evaluation::GuardrailMetric::TaggedSymbol
-          )
-        COMPLETENESS =
-          T.let(
-            :completeness,
-            Deeprails::Evaluation::GuardrailMetric::TaggedSymbol
-          )
-        INSTRUCTION_ADHERENCE =
-          T.let(
-            :instruction_adherence,
-            Deeprails::Evaluation::GuardrailMetric::TaggedSymbol
-          )
-        CONTEXT_ADHERENCE =
-          T.let(
-            :context_adherence,
-            Deeprails::Evaluation::GuardrailMetric::TaggedSymbol
-          )
-        GROUND_TRUTH_ADHERENCE =
-          T.let(
-            :ground_truth_adherence,
-            Deeprails::Evaluation::GuardrailMetric::TaggedSymbol
-          )
-        COMPREHENSIVE_SAFETY =
-          T.let(
-            :comprehensive_safety,
-            Deeprails::Evaluation::GuardrailMetric::TaggedSymbol
-          )
-
-        sig do
-          override.returns(
-            T::Array[Deeprails::Evaluation::GuardrailMetric::TaggedSymbol]
-          )
-        end
-        def self.values
-        end
-      end
-    end
-  end
-end

data/rbi/deeprails/resources/evaluate.rbi

@@ -1,66 +0,0 @@
-# typed: strong
-
-module Deeprails
-  module Resources
-    class Evaluate
-      # Use this endpoint to evaluate a model's input and output pair against selected
-      # guardrail metrics
-      sig do
-        params(
-          model_input: Deeprails::EvaluateCreateParams::ModelInput::OrHash,
-          model_output: String,
-          run_mode: Deeprails::EvaluateCreateParams::RunMode::OrSymbol,
-          guardrail_metrics:
-            T::Array[
-              Deeprails::EvaluateCreateParams::GuardrailMetric::OrSymbol
-            ],
-          model_used: String,
-          nametag: String,
-          request_options: Deeprails::RequestOptions::OrHash
-        ).returns(Deeprails::Evaluation)
-      end
-      def create(
-        # A dictionary of inputs sent to the LLM to generate output. The dictionary must
-        # contain at least a `user_prompt` field or a `system_prompt` field. For
-        # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
-        model_input:,
-        # Output generated by the LLM to be evaluated.
-        model_output:,
-        # Run mode for the evaluation. The run mode allows the user to optimize for speed,
-        # accuracy, and cost by determining which models are used to evaluate the event.
-        # Available run modes include `precision_plus`, `precision`, `smart`, and
-        # `economy`. Defaults to `smart`.
-        run_mode:,
-        # An array of guardrail metrics that the model input and output pair will be
-        # evaluated on. For non-enterprise users, these will be limited to the allowed
-        # guardrail metrics.
-        guardrail_metrics: nil,
-        # Model ID used to generate the output, like `gpt-4o` or `o3`.
-        model_used: nil,
-        # An optional, user-defined tag for the evaluation.
-        nametag: nil,
-        request_options: {}
-      )
-      end
-
-      # Use this endpoint to retrieve the evaluation record for a given evaluation ID
-      sig do
-        params(
-          eval_id: String,
-          request_options: Deeprails::RequestOptions::OrHash
-        ).returns(Deeprails::Evaluation)
-      end
-      def retrieve(
-        # The ID of the evaluation to retrieve.
-        eval_id,
-        request_options: {}
-      )
-      end
-
-      # @api private
-      sig { params(client: Deeprails::Client).returns(T.attached_class) }
-      def self.new(client:)
-      end
-    end
-  end
-end

data/sig/deeprails/models/evaluate_create_params.rbs

@@ -1,122 +0,0 @@
-module Deeprails
-  module Models
-    type evaluate_create_params =
-      {
-        model_input: Deeprails::EvaluateCreateParams::ModelInput,
-        model_output: String,
-        run_mode: Deeprails::Models::EvaluateCreateParams::run_mode,
-        guardrail_metrics: ::Array[Deeprails::Models::EvaluateCreateParams::guardrail_metric],
-        model_used: String,
-        nametag: String
-      }
-      & Deeprails::Internal::Type::request_parameters
-
-    class EvaluateCreateParams < Deeprails::Internal::Type::BaseModel
-      extend Deeprails::Internal::Type::RequestParameters::Converter
-      include Deeprails::Internal::Type::RequestParameters
-
-      attr_accessor model_input: Deeprails::EvaluateCreateParams::ModelInput
-
-      attr_accessor model_output: String
-
-      attr_accessor run_mode: Deeprails::Models::EvaluateCreateParams::run_mode
-
-      attr_reader guardrail_metrics: ::Array[Deeprails::Models::EvaluateCreateParams::guardrail_metric]?
-
-      def guardrail_metrics=: (
-        ::Array[Deeprails::Models::EvaluateCreateParams::guardrail_metric]
-      ) -> ::Array[Deeprails::Models::EvaluateCreateParams::guardrail_metric]
-
-      attr_reader model_used: String?
-
-      def model_used=: (String) -> String
-
-      attr_reader nametag: String?
-
-      def nametag=: (String) -> String
-
-      def initialize: (
-        model_input: Deeprails::EvaluateCreateParams::ModelInput,
-        model_output: String,
-        run_mode: Deeprails::Models::EvaluateCreateParams::run_mode,
-        ?guardrail_metrics: ::Array[Deeprails::Models::EvaluateCreateParams::guardrail_metric],
-        ?model_used: String,
-        ?nametag: String,
-        ?request_options: Deeprails::request_opts
-      ) -> void
-
-      def to_hash: -> {
-        model_input: Deeprails::EvaluateCreateParams::ModelInput,
-        model_output: String,
-        run_mode: Deeprails::Models::EvaluateCreateParams::run_mode,
-        guardrail_metrics: ::Array[Deeprails::Models::EvaluateCreateParams::guardrail_metric],
-        model_used: String,
-        nametag: String,
-        request_options: Deeprails::RequestOptions
-      }
-
-      type model_input =
-        { ground_truth: String, system_prompt: String, user_prompt: String }
-
-      class ModelInput < Deeprails::Internal::Type::BaseModel
-        attr_reader ground_truth: String?
-
-        def ground_truth=: (String) -> String
-
-        attr_reader system_prompt: String?
-
-        def system_prompt=: (String) -> String
-
-        attr_reader user_prompt: String?
-
-        def user_prompt=: (String) -> String
-
-        def initialize: (
-          ?ground_truth: String,
-          ?system_prompt: String,
-          ?user_prompt: String
-        ) -> void
-
-        def to_hash: -> {
-          ground_truth: String,
-          system_prompt: String,
-          user_prompt: String
-        }
-      end
-
-      type run_mode = :precision_plus | :precision | :smart | :economy
-
-      module RunMode
-        extend Deeprails::Internal::Type::Enum
-
-        PRECISION_PLUS: :precision_plus
-        PRECISION: :precision
-        SMART: :smart
-        ECONOMY: :economy
-
-        def self?.values: -> ::Array[Deeprails::Models::EvaluateCreateParams::run_mode]
-      end
-
-      type guardrail_metric =
-        :correctness
-        | :completeness
-        | :instruction_adherence
-        | :context_adherence
-        | :ground_truth_adherence
-        | :comprehensive_safety
-
-      module GuardrailMetric
-        extend Deeprails::Internal::Type::Enum
-
-        CORRECTNESS: :correctness
-        COMPLETENESS: :completeness
-        INSTRUCTION_ADHERENCE: :instruction_adherence
-        CONTEXT_ADHERENCE: :context_adherence
-        GROUND_TRUTH_ADHERENCE: :ground_truth_adherence
-        COMPREHENSIVE_SAFETY: :comprehensive_safety
-
-        def self?.values: -> ::Array[Deeprails::Models::EvaluateCreateParams::guardrail_metric]
-      end
-    end
-  end
-end

data/sig/deeprails/models/evaluate_retrieve_params.rbs

@@ -1,15 +0,0 @@
-module Deeprails
-  module Models
-    type evaluate_retrieve_params =
-      {  } & Deeprails::Internal::Type::request_parameters
-
-    class EvaluateRetrieveParams < Deeprails::Internal::Type::BaseModel
-      extend Deeprails::Internal::Type::RequestParameters::Converter
-      include Deeprails::Internal::Type::RequestParameters
-
-      def initialize: (?request_options: Deeprails::request_opts) -> void
-
-      def to_hash: -> { request_options: Deeprails::RequestOptions }
-    end
-  end
-end