deeprails 0.1.0

data/rbi/deeprails/models/evaluation.rbi

@@ -0,0 +1,382 @@
+# typed: strong
+
+module Deeprails
+  module Models
+    class Evaluation < Deeprails::Internal::Type::BaseModel
+      OrHash =
+        T.type_alias do
+          T.any(Deeprails::Evaluation, Deeprails::Internal::AnyHash)
+        end
+
+      # A unique evaluation ID.
+      sig { returns(String) }
+      attr_accessor :eval_id
+
+      # Status of the evaluation.
+      sig { returns(Deeprails::Evaluation::EvaluationStatus::TaggedSymbol) }
+      attr_accessor :evaluation_status
+
+      # A dictionary of inputs sent to the LLM to generate output. The dictionary must
+      # contain a `user_prompt` field and an optional `context` field. Additional
+      # properties are allowed.
+      sig { returns(Deeprails::Evaluation::ModelInput) }
+      attr_reader :model_input
+
+      sig do
+        params(model_input: Deeprails::Evaluation::ModelInput::OrHash).void
+      end
+      attr_writer :model_input
+
+      # Output generated by the LLM to be evaluated.
+      sig { returns(String) }
+      attr_accessor :model_output
+
+      # Run mode for the evaluation. The run mode allows the user to optimize for speed,
+      # accuracy, and cost by determining which models are used to evaluate the event.
+      sig { returns(Deeprails::Evaluation::RunMode::TaggedSymbol) }
+      attr_accessor :run_mode
+
+      # The time the evaluation was created in UTC.
+      sig { returns(T.nilable(Time)) }
+      attr_reader :created_at
+
+      sig { params(created_at: Time).void }
+      attr_writer :created_at
+
+      # The time the evaluation completed in UTC.
+      sig { returns(T.nilable(Time)) }
+      attr_reader :end_timestamp
+
+      sig { params(end_timestamp: Time).void }
+      attr_writer :end_timestamp
+
+      # Description of the error causing the evaluation to fail, if any.
+      sig { returns(T.nilable(String)) }
+      attr_reader :error_message
+
+      sig { params(error_message: String).void }
+      attr_writer :error_message
+
+      # The time the error causing the evaluation to fail was recorded.
+      sig { returns(T.nilable(Time)) }
+      attr_reader :error_timestamp
+
+      sig { params(error_timestamp: Time).void }
+      attr_writer :error_timestamp
+
+      # Evaluation result consisting of average scores and rationales for each of the
+      # evaluated guardrail metrics.
+      sig { returns(T.nilable(T::Hash[Symbol, T.anything])) }
+      attr_reader :evaluation_result
+
+      sig { params(evaluation_result: T::Hash[Symbol, T.anything]).void }
+      attr_writer :evaluation_result
+
+      # Total cost of the evaluation.
+      sig { returns(T.nilable(Float)) }
+      attr_reader :evaluation_total_cost
+
+      sig { params(evaluation_total_cost: Float).void }
+      attr_writer :evaluation_total_cost
+
+      # An array of guardrail metrics that the model input and output pair will be
+      # evaluated on.
+      sig do
+        returns(
+          T.nilable(
+            T::Array[Deeprails::Evaluation::GuardrailMetric::TaggedSymbol]
+          )
+        )
+      end
+      attr_reader :guardrail_metrics
+
+      sig do
+        params(
+          guardrail_metrics:
+            T::Array[Deeprails::Evaluation::GuardrailMetric::OrSymbol]
+        ).void
+      end
+      attr_writer :guardrail_metrics
+
+      # Model ID used to generate the output, like `gpt-4o` or `o3`.
+      sig { returns(T.nilable(String)) }
+      attr_reader :model_used
+
+      sig { params(model_used: String).void }
+      attr_writer :model_used
+
+      # The most recent time the evaluation was modified in UTC.
+      sig { returns(T.nilable(Time)) }
+      attr_reader :modified_at
+
+      sig { params(modified_at: Time).void }
+      attr_writer :modified_at
+
+      # An optional, user-defined tag for the evaluation.
+      sig { returns(T.nilable(String)) }
+      attr_reader :nametag
+
+      sig { params(nametag: String).void }
+      attr_writer :nametag
+
+      # Evaluation progress. Values range between 0 and 100; 100 corresponds to a
+      # completed `evaluation_status`.
+      sig { returns(T.nilable(Integer)) }
+      attr_reader :progress
+
+      sig { params(progress: Integer).void }
+      attr_writer :progress
+
+      # The time the evaluation started in UTC.
+      sig { returns(T.nilable(Time)) }
+      attr_reader :start_timestamp
+
+      sig { params(start_timestamp: Time).void }
+      attr_writer :start_timestamp
+
+      sig do
+        params(
+          eval_id: String,
+          evaluation_status: Deeprails::Evaluation::EvaluationStatus::OrSymbol,
+          model_input: Deeprails::Evaluation::ModelInput::OrHash,
+          model_output: String,
+          run_mode: Deeprails::Evaluation::RunMode::OrSymbol,
+          created_at: Time,
+          end_timestamp: Time,
+          error_message: String,
+          error_timestamp: Time,
+          evaluation_result: T::Hash[Symbol, T.anything],
+          evaluation_total_cost: Float,
+          guardrail_metrics:
+            T::Array[Deeprails::Evaluation::GuardrailMetric::OrSymbol],
+          model_used: String,
+          modified_at: Time,
+          nametag: String,
+          progress: Integer,
+          start_timestamp: Time
+        ).returns(T.attached_class)
+      end
+      def self.new(
+        # A unique evaluation ID.
+        eval_id:,
+        # Status of the evaluation.
+        evaluation_status:,
+        # A dictionary of inputs sent to the LLM to generate output. The dictionary must
+        # contain a `user_prompt` field and an optional `context` field. Additional
+        # properties are allowed.
+        model_input:,
+        # Output generated by the LLM to be evaluated.
+        model_output:,
+        # Run mode for the evaluation. The run mode allows the user to optimize for speed,
+        # accuracy, and cost by determining which models are used to evaluate the event.
+        run_mode:,
+        # The time the evaluation was created in UTC.
+        created_at: nil,
+        # The time the evaluation completed in UTC.
+        end_timestamp: nil,
+        # Description of the error causing the evaluation to fail, if any.
+        error_message: nil,
+        # The time the error causing the evaluation to fail was recorded.
+        error_timestamp: nil,
+        # Evaluation result consisting of average scores and rationales for each of the
+        # evaluated guardrail metrics.
+        evaluation_result: nil,
+        # Total cost of the evaluation.
+        evaluation_total_cost: nil,
+        # An array of guardrail metrics that the model input and output pair will be
+        # evaluated on.
+        guardrail_metrics: nil,
+        # Model ID used to generate the output, like `gpt-4o` or `o3`.
+        model_used: nil,
+        # The most recent time the evaluation was modified in UTC.
+        modified_at: nil,
+        # An optional, user-defined tag for the evaluation.
+        nametag: nil,
+        # Evaluation progress. Values range between 0 and 100; 100 corresponds to a
+        # completed `evaluation_status`.
+        progress: nil,
+        # The time the evaluation started in UTC.
+        start_timestamp: nil
+      )
+      end
+
+      sig do
+        override.returns(
+          {
+            eval_id: String,
+            evaluation_status:
+              Deeprails::Evaluation::EvaluationStatus::TaggedSymbol,
+            model_input: Deeprails::Evaluation::ModelInput,
+            model_output: String,
+            run_mode: Deeprails::Evaluation::RunMode::TaggedSymbol,
+            created_at: Time,
+            end_timestamp: Time,
+            error_message: String,
+            error_timestamp: Time,
+            evaluation_result: T::Hash[Symbol, T.anything],
+            evaluation_total_cost: Float,
+            guardrail_metrics:
+              T::Array[Deeprails::Evaluation::GuardrailMetric::TaggedSymbol],
+            model_used: String,
+            modified_at: Time,
+            nametag: String,
+            progress: Integer,
+            start_timestamp: Time
+          }
+        )
+      end
+      def to_hash
+      end
+
+      # Status of the evaluation.
+      module EvaluationStatus
+        extend Deeprails::Internal::Type::Enum
+
+        TaggedSymbol =
+          T.type_alias do
+            T.all(Symbol, Deeprails::Evaluation::EvaluationStatus)
+          end
+        OrSymbol = T.type_alias { T.any(Symbol, String) }
+
+        IN_PROGRESS =
+          T.let(
+            :in_progress,
+            Deeprails::Evaluation::EvaluationStatus::TaggedSymbol
+          )
+        COMPLETED =
+          T.let(
+            :completed,
+            Deeprails::Evaluation::EvaluationStatus::TaggedSymbol
+          )
+        CANCELED =
+          T.let(
+            :canceled,
+            Deeprails::Evaluation::EvaluationStatus::TaggedSymbol
+          )
+        QUEUED =
+          T.let(:queued, Deeprails::Evaluation::EvaluationStatus::TaggedSymbol)
+        FAILED =
+          T.let(:failed, Deeprails::Evaluation::EvaluationStatus::TaggedSymbol)
+
+        sig do
+          override.returns(
+            T::Array[Deeprails::Evaluation::EvaluationStatus::TaggedSymbol]
+          )
+        end
+        def self.values
+        end
+      end
+
+      class ModelInput < Deeprails::Internal::Type::BaseModel
+        OrHash =
+          T.type_alias do
+            T.any(
+              Deeprails::Evaluation::ModelInput,
+              Deeprails::Internal::AnyHash
+            )
+          end
+
+        # The user prompt used to generate the output.
+        sig { returns(String) }
+        attr_accessor :user_prompt
+
+        # Optional context supplied to the LLM when generating the output.
+        sig { returns(T.nilable(String)) }
+        attr_reader :context
+
+        sig { params(context: String).void }
+        attr_writer :context
+
+        # A dictionary of inputs sent to the LLM to generate output. The dictionary must
+        # contain a `user_prompt` field and an optional `context` field. Additional
+        # properties are allowed.
+        sig do
+          params(user_prompt: String, context: String).returns(T.attached_class)
+        end
+        def self.new(
+          # The user prompt used to generate the output.
+          user_prompt:,
+          # Optional context supplied to the LLM when generating the output.
+          context: nil
+        )
+        end
+
+        sig { override.returns({ user_prompt: String, context: String }) }
+        def to_hash
+        end
+      end
+
+      # Run mode for the evaluation. The run mode allows the user to optimize for speed,
+      # accuracy, and cost by determining which models are used to evaluate the event.
+      module RunMode
+        extend Deeprails::Internal::Type::Enum
+
+        TaggedSymbol =
+          T.type_alias { T.all(Symbol, Deeprails::Evaluation::RunMode) }
+        OrSymbol = T.type_alias { T.any(Symbol, String) }
+
+        PRECISION_PLUS =
+          T.let(:precision_plus, Deeprails::Evaluation::RunMode::TaggedSymbol)
+        PRECISION =
+          T.let(:precision, Deeprails::Evaluation::RunMode::TaggedSymbol)
+        SMART = T.let(:smart, Deeprails::Evaluation::RunMode::TaggedSymbol)
+        ECONOMY = T.let(:economy, Deeprails::Evaluation::RunMode::TaggedSymbol)
+
+        sig do
+          override.returns(
+            T::Array[Deeprails::Evaluation::RunMode::TaggedSymbol]
+          )
+        end
+        def self.values
+        end
+      end
+
+      module GuardrailMetric
+        extend Deeprails::Internal::Type::Enum
+
+        TaggedSymbol =
+          T.type_alias { T.all(Symbol, Deeprails::Evaluation::GuardrailMetric) }
+        OrSymbol = T.type_alias { T.any(Symbol, String) }
+
+        CORRECTNESS =
+          T.let(
+            :correctness,
+            Deeprails::Evaluation::GuardrailMetric::TaggedSymbol
+          )
+        COMPLETENESS =
+          T.let(
+            :completeness,
+            Deeprails::Evaluation::GuardrailMetric::TaggedSymbol
+          )
+        INSTRUCTION_ADHERENCE =
+          T.let(
+            :instruction_adherence,
+            Deeprails::Evaluation::GuardrailMetric::TaggedSymbol
+          )
+        CONTEXT_ADHERENCE =
+          T.let(
+            :context_adherence,
+            Deeprails::Evaluation::GuardrailMetric::TaggedSymbol
+          )
+        GROUND_TRUTH_ADHERENCE =
+          T.let(
+            :ground_truth_adherence,
+            Deeprails::Evaluation::GuardrailMetric::TaggedSymbol
+          )
+        COMPREHENSIVE_SAFETY =
+          T.let(
+            :comprehensive_safety,
+            Deeprails::Evaluation::GuardrailMetric::TaggedSymbol
+          )
+
+        sig do
+          override.returns(
+            T::Array[Deeprails::Evaluation::GuardrailMetric::TaggedSymbol]
+          )
+        end
+        def self.values
+        end
+      end
+    end
+  end
+end

data/rbi/deeprails/models/monitor_create_params.rbi

@@ -0,0 +1,54 @@
+# typed: strong
+
+module Deeprails
+  module Models
+    class MonitorCreateParams < Deeprails::Internal::Type::BaseModel
+      extend Deeprails::Internal::Type::RequestParameters::Converter
+      include Deeprails::Internal::Type::RequestParameters
+
+      OrHash =
+        T.type_alias do
+          T.any(Deeprails::MonitorCreateParams, Deeprails::Internal::AnyHash)
+        end
+
+      # Name of the new monitor.
+      sig { returns(String) }
+      attr_accessor :name
+
+      # Description of the new monitor.
+      sig { returns(T.nilable(String)) }
+      attr_reader :description
+
+      sig { params(description: String).void }
+      attr_writer :description
+
+      sig do
+        params(
+          name: String,
+          description: String,
+          request_options: Deeprails::RequestOptions::OrHash
+        ).returns(T.attached_class)
+      end
+      def self.new(
+        # Name of the new monitor.
+        name:,
+        # Description of the new monitor.
+        description: nil,
+        request_options: {}
+      )
+      end
+
+      sig do
+        override.returns(
+          {
+            name: String,
+            description: String,
+            request_options: Deeprails::RequestOptions
+          }
+        )
+      end
+      def to_hash
+      end
+    end
+  end
+end

data/rbi/deeprails/models/monitor_retrieve_params.rbi

@@ -0,0 +1,43 @@
+# typed: strong
+
+module Deeprails
+  module Models
+    class MonitorRetrieveParams < Deeprails::Internal::Type::BaseModel
+      extend Deeprails::Internal::Type::RequestParameters::Converter
+      include Deeprails::Internal::Type::RequestParameters
+
+      OrHash =
+        T.type_alias do
+          T.any(Deeprails::MonitorRetrieveParams, Deeprails::Internal::AnyHash)
+        end
+
+      # Limit the returned events associated with this monitor. Defaults to 10.
+      sig { returns(T.nilable(Integer)) }
+      attr_reader :limit
+
+      sig { params(limit: Integer).void }
+      attr_writer :limit
+
+      sig do
+        params(
+          limit: Integer,
+          request_options: Deeprails::RequestOptions::OrHash
+        ).returns(T.attached_class)
+      end
+      def self.new(
+        # Limit the returned events associated with this monitor. Defaults to 10.
+        limit: nil,
+        request_options: {}
+      )
+      end
+
+      sig do
+        override.returns(
+          { limit: Integer, request_options: Deeprails::RequestOptions }
+        )
+      end
+      def to_hash
+      end
+    end
+  end
+end