deeprails 0.8.0 → 0.10.0

data/rbi/deeprails/models/file_response.rbi

@@ -0,0 +1,84 @@
+# typed: strong
+
+module Deeprails
+  module Models
+    class FileResponse < Deeprails::Internal::Type::BaseModel
+      OrHash =
+        T.type_alias do
+          T.any(Deeprails::FileResponse, Deeprails::Internal::AnyHash)
+        end
+
+      # The time the file was created in UTC.
+      sig { returns(T.nilable(Time)) }
+      attr_reader :created_at
+
+      sig { params(created_at: Time).void }
+      attr_writer :created_at
+
+      # A unique file ID.
+      sig { returns(T.nilable(String)) }
+      attr_reader :file_id
+
+      sig { params(file_id: String).void }
+      attr_writer :file_id
+
+      # Name of the file.
+      sig { returns(T.nilable(String)) }
+      attr_reader :file_name
+
+      sig { params(file_name: String).void }
+      attr_writer :file_name
+
+      # Path to the s3 bucket where the file is stored.
+      sig { returns(T.nilable(String)) }
+      attr_reader :file_path
+
+      sig { params(file_path: String).void }
+      attr_writer :file_path
+
+      # The most recent time the file was modified in UTC.
+      sig { returns(T.nilable(Time)) }
+      attr_reader :updated_at
+
+      sig { params(updated_at: Time).void }
+      attr_writer :updated_at
+
+      sig do
+        params(
+          created_at: Time,
+          file_id: String,
+          file_name: String,
+          file_path: String,
+          updated_at: Time
+        ).returns(T.attached_class)
+      end
+      def self.new(
+        # The time the file was created in UTC.
+        created_at: nil,
+        # A unique file ID.
+        file_id: nil,
+        # Name of the file.
+        file_name: nil,
+        # Path to the s3 bucket where the file is stored.
+        file_path: nil,
+        # The most recent time the file was modified in UTC.
+        updated_at: nil
+      )
+      end
+
+      sig do
+        override.returns(
+          {
+            created_at: Time,
+            file_id: String,
+            file_name: String,
+            file_path: String,
+            updated_at: Time
+          }
+        )
+      end
+      def to_hash
+      end
+    end
+  end
+end

data/rbi/deeprails/models/file_upload_params.rbi

@@ -0,0 +1,43 @@
+# typed: strong
+
+module Deeprails
+  module Models
+    class FileUploadParams < Deeprails::Internal::Type::BaseModel
+      extend Deeprails::Internal::Type::RequestParameters::Converter
+      include Deeprails::Internal::Type::RequestParameters
+
+      OrHash =
+        T.type_alias do
+          T.any(Deeprails::FileUploadParams, Deeprails::Internal::AnyHash)
+        end
+
+      # The contents of the file to upload.
+      sig { returns(Deeprails::Internal::FileInput) }
+      attr_accessor :file
+
+      sig do
+        params(
+          file: Deeprails::Internal::FileInput,
+          request_options: Deeprails::RequestOptions::OrHash
+        ).returns(T.attached_class)
+      end
+      def self.new(
+        # The contents of the file to upload.
+        file:,
+        request_options: {}
+      )
+      end
+
+      sig do
+        override.returns(
+          {
+            file: Deeprails::Internal::FileInput,
+            request_options: Deeprails::RequestOptions
+          }
+        )
+      end
+      def to_hash
+      end
+    end
+  end
+end

data/rbi/deeprails/models/monitor_detail_response.rbi

@@ -39,10 +39,19 @@ module Deeprails
 
       # An array of all evaluations performed by this monitor. Each one corresponds to a
       # separate monitor event.
-      sig { returns(T.nilable(T::Array[Deeprails::Evaluation])) }
+      sig do
+        returns(
+          T.nilable(T::Array[Deeprails::MonitorDetailResponse::Evaluation])
+        )
+      end
       attr_reader :evaluations
 
-      sig { params(evaluations: T::Array[Deeprails::Evaluation::OrHash]).void }
+      sig do
+        params(
+          evaluations:
+            T::Array[Deeprails::MonitorDetailResponse::Evaluation::OrHash]
+        ).void
+      end
       attr_writer :evaluations
 
       # Contains five fields used for stats of this monitor: total evaluations,
@@ -78,7 +87,8 @@ module Deeprails
           name: String,
           created_at: Time,
           description: String,
-          evaluations: T::Array[Deeprails::Evaluation::OrHash],
+          evaluations:
+            T::Array[Deeprails::MonitorDetailResponse::Evaluation::OrHash],
           stats: Deeprails::MonitorDetailResponse::Stats::OrHash,
           updated_at: Time,
           user_id: String
@@ -119,7 +129,7 @@ module Deeprails
             name: String,
             created_at: Time,
             description: String,
-            evaluations: T::Array[Deeprails::Evaluation],
+            evaluations: T::Array[Deeprails::MonitorDetailResponse::Evaluation],
             stats: Deeprails::MonitorDetailResponse::Stats,
             updated_at: Time,
             user_id: String
@@ -162,6 +172,475 @@ module Deeprails
         end
       end
 
+      class Evaluation < Deeprails::Internal::Type::BaseModel
+        OrHash =
+          T.type_alias do
+            T.any(
+              Deeprails::MonitorDetailResponse::Evaluation,
+              Deeprails::Internal::AnyHash
+            )
+          end
+
+        # A unique evaluation ID.
+        sig { returns(String) }
+        attr_accessor :eval_id
+
+        # Status of the evaluation.
+        sig do
+          returns(
+            Deeprails::MonitorDetailResponse::Evaluation::EvaluationStatus::TaggedSymbol
+          )
+        end
+        attr_accessor :evaluation_status
+
+        # A dictionary of inputs sent to the LLM to generate output. The dictionary must
+        # contain at least a `user_prompt` field or a `system_prompt` field. For
+        # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
+        sig do
+          returns(Deeprails::MonitorDetailResponse::Evaluation::ModelInput)
+        end
+        attr_reader :model_input
+
+        sig do
+          params(
+            model_input:
+              Deeprails::MonitorDetailResponse::Evaluation::ModelInput::OrHash
+          ).void
+        end
+        attr_writer :model_input
+
+        # Output generated by the LLM to be evaluated.
+        sig { returns(String) }
+        attr_accessor :model_output
+
+        # Run mode for the evaluation. The run mode allows the user to optimize for speed,
+        # accuracy, and cost by determining which models are used to evaluate the event.
+        sig do
+          returns(
+            Deeprails::MonitorDetailResponse::Evaluation::RunMode::TaggedSymbol
+          )
+        end
+        attr_accessor :run_mode
+
+        # The time the evaluation was created in UTC.
+        sig { returns(T.nilable(Time)) }
+        attr_reader :created_at
+
+        sig { params(created_at: Time).void }
+        attr_writer :created_at
+
+        # The time the evaluation completed in UTC.
+        sig { returns(T.nilable(Time)) }
+        attr_reader :end_timestamp
+
+        sig { params(end_timestamp: Time).void }
+        attr_writer :end_timestamp
+
+        # Description of the error causing the evaluation to fail, if any.
+        sig { returns(T.nilable(String)) }
+        attr_reader :error_message
+
+        sig { params(error_message: String).void }
+        attr_writer :error_message
+
+        # The time the error causing the evaluation to fail was recorded.
+        sig { returns(T.nilable(Time)) }
+        attr_reader :error_timestamp
+
+        sig { params(error_timestamp: Time).void }
+        attr_writer :error_timestamp
+
+        # Evaluation result consisting of average scores and rationales for each of the
+        # evaluated guardrail metrics.
+        sig { returns(T.nilable(T::Hash[Symbol, T.anything])) }
+        attr_reader :evaluation_result
+
+        sig { params(evaluation_result: T::Hash[Symbol, T.anything]).void }
+        attr_writer :evaluation_result
+
+        # Total cost of the evaluation.
+        sig { returns(T.nilable(Float)) }
+        attr_reader :evaluation_total_cost
+
+        sig { params(evaluation_total_cost: Float).void }
+        attr_writer :evaluation_total_cost
+
+        # An array of guardrail metrics that the model input and output pair will be
+        # evaluated on.
+        sig do
+          returns(
+            T.nilable(
+              T::Array[
+                Deeprails::MonitorDetailResponse::Evaluation::GuardrailMetric::TaggedSymbol
+              ]
+            )
+          )
+        end
+        attr_reader :guardrail_metrics
+
+        sig do
+          params(
+            guardrail_metrics:
+              T::Array[
+                Deeprails::MonitorDetailResponse::Evaluation::GuardrailMetric::OrSymbol
+              ]
+          ).void
+        end
+        attr_writer :guardrail_metrics
+
+        # Model ID used to generate the output, like `gpt-4o` or `o3`.
+        sig { returns(T.nilable(String)) }
+        attr_reader :model_used
+
+        sig { params(model_used: String).void }
+        attr_writer :model_used
+
+        # The most recent time the evaluation was modified in UTC.
+        sig { returns(T.nilable(Time)) }
+        attr_reader :modified_at
+
+        sig { params(modified_at: Time).void }
+        attr_writer :modified_at
+
+        # An optional, user-defined tag for the evaluation.
+        sig { returns(T.nilable(String)) }
+        attr_reader :nametag
+
+        sig { params(nametag: String).void }
+        attr_writer :nametag
+
+        # Evaluation progress. Values range between 0 and 100; 100 corresponds to a
+        # completed `evaluation_status`.
+        sig { returns(T.nilable(Integer)) }
+        attr_reader :progress
+
+        sig { params(progress: Integer).void }
+        attr_writer :progress
+
+        # The time the evaluation started in UTC.
+        sig { returns(T.nilable(Time)) }
+        attr_reader :start_timestamp
+
+        sig { params(start_timestamp: Time).void }
+        attr_writer :start_timestamp
+
+        sig do
+          params(
+            eval_id: String,
+            evaluation_status:
+              Deeprails::MonitorDetailResponse::Evaluation::EvaluationStatus::OrSymbol,
+            model_input:
+              Deeprails::MonitorDetailResponse::Evaluation::ModelInput::OrHash,
+            model_output: String,
+            run_mode:
+              Deeprails::MonitorDetailResponse::Evaluation::RunMode::OrSymbol,
+            created_at: Time,
+            end_timestamp: Time,
+            error_message: String,
+            error_timestamp: Time,
+            evaluation_result: T::Hash[Symbol, T.anything],
+            evaluation_total_cost: Float,
+            guardrail_metrics:
+              T::Array[
+                Deeprails::MonitorDetailResponse::Evaluation::GuardrailMetric::OrSymbol
+              ],
+            model_used: String,
+            modified_at: Time,
+            nametag: String,
+            progress: Integer,
+            start_timestamp: Time
+          ).returns(T.attached_class)
+        end
+        def self.new(
+          # A unique evaluation ID.
+          eval_id:,
+          # Status of the evaluation.
+          evaluation_status:,
+          # A dictionary of inputs sent to the LLM to generate output. The dictionary must
+          # contain at least a `user_prompt` field or a `system_prompt` field. For
+          # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
+          model_input:,
+          # Output generated by the LLM to be evaluated.
+          model_output:,
+          # Run mode for the evaluation. The run mode allows the user to optimize for speed,
+          # accuracy, and cost by determining which models are used to evaluate the event.
+          run_mode:,
+          # The time the evaluation was created in UTC.
+          created_at: nil,
+          # The time the evaluation completed in UTC.
+          end_timestamp: nil,
+          # Description of the error causing the evaluation to fail, if any.
+          error_message: nil,
+          # The time the error causing the evaluation to fail was recorded.
+          error_timestamp: nil,
+          # Evaluation result consisting of average scores and rationales for each of the
+          # evaluated guardrail metrics.
+          evaluation_result: nil,
+          # Total cost of the evaluation.
+          evaluation_total_cost: nil,
+          # An array of guardrail metrics that the model input and output pair will be
+          # evaluated on.
+          guardrail_metrics: nil,
+          # Model ID used to generate the output, like `gpt-4o` or `o3`.
+          model_used: nil,
+          # The most recent time the evaluation was modified in UTC.
+          modified_at: nil,
+          # An optional, user-defined tag for the evaluation.
+          nametag: nil,
+          # Evaluation progress. Values range between 0 and 100; 100 corresponds to a
+          # completed `evaluation_status`.
+          progress: nil,
+          # The time the evaluation started in UTC.
+          start_timestamp: nil
+        )
+        end
+
+        sig do
+          override.returns(
+            {
+              eval_id: String,
+              evaluation_status:
+                Deeprails::MonitorDetailResponse::Evaluation::EvaluationStatus::TaggedSymbol,
+              model_input:
+                Deeprails::MonitorDetailResponse::Evaluation::ModelInput,
+              model_output: String,
+              run_mode:
+                Deeprails::MonitorDetailResponse::Evaluation::RunMode::TaggedSymbol,
+              created_at: Time,
+              end_timestamp: Time,
+              error_message: String,
+              error_timestamp: Time,
+              evaluation_result: T::Hash[Symbol, T.anything],
+              evaluation_total_cost: Float,
+              guardrail_metrics:
+                T::Array[
+                  Deeprails::MonitorDetailResponse::Evaluation::GuardrailMetric::TaggedSymbol
+                ],
+              model_used: String,
+              modified_at: Time,
+              nametag: String,
+              progress: Integer,
+              start_timestamp: Time
+            }
+          )
+        end
+        def to_hash
+        end
+
+        # Status of the evaluation.
+        module EvaluationStatus
+          extend Deeprails::Internal::Type::Enum
+
+          TaggedSymbol =
+            T.type_alias do
+              T.all(
+                Symbol,
+                Deeprails::MonitorDetailResponse::Evaluation::EvaluationStatus
+              )
+            end
+          OrSymbol = T.type_alias { T.any(Symbol, String) }
+
+          IN_PROGRESS =
+            T.let(
+              :in_progress,
+              Deeprails::MonitorDetailResponse::Evaluation::EvaluationStatus::TaggedSymbol
+            )
+          COMPLETED =
+            T.let(
+              :completed,
+              Deeprails::MonitorDetailResponse::Evaluation::EvaluationStatus::TaggedSymbol
+            )
+          CANCELED =
+            T.let(
+              :canceled,
+              Deeprails::MonitorDetailResponse::Evaluation::EvaluationStatus::TaggedSymbol
+            )
+          QUEUED =
+            T.let(
+              :queued,
+              Deeprails::MonitorDetailResponse::Evaluation::EvaluationStatus::TaggedSymbol
+            )
+          FAILED =
+            T.let(
+              :failed,
+              Deeprails::MonitorDetailResponse::Evaluation::EvaluationStatus::TaggedSymbol
+            )
+
+          sig do
+            override.returns(
+              T::Array[
+                Deeprails::MonitorDetailResponse::Evaluation::EvaluationStatus::TaggedSymbol
+              ]
+            )
+          end
+          def self.values
+          end
+        end
+
+        class ModelInput < Deeprails::Internal::Type::BaseModel
+          OrHash =
+            T.type_alias do
+              T.any(
+                Deeprails::MonitorDetailResponse::Evaluation::ModelInput,
+                Deeprails::Internal::AnyHash
+              )
+            end
+
+          # The ground truth for evaluating Ground Truth Adherence guardrail.
+          sig { returns(T.nilable(String)) }
+          attr_reader :ground_truth
+
+          sig { params(ground_truth: String).void }
+          attr_writer :ground_truth
+
+          # The system prompt used to generate the output.
+          sig { returns(T.nilable(String)) }
+          attr_reader :system_prompt
+
+          sig { params(system_prompt: String).void }
+          attr_writer :system_prompt
+
+          # The user prompt used to generate the output.
+          sig { returns(T.nilable(String)) }
+          attr_reader :user_prompt
+
+          sig { params(user_prompt: String).void }
+          attr_writer :user_prompt
+
+          # A dictionary of inputs sent to the LLM to generate output. The dictionary must
+          # contain at least a `user_prompt` field or a `system_prompt` field. For
+          # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
+          sig do
+            params(
+              ground_truth: String,
+              system_prompt: String,
+              user_prompt: String
+            ).returns(T.attached_class)
+          end
+          def self.new(
+            # The ground truth for evaluating Ground Truth Adherence guardrail.
+            ground_truth: nil,
+            # The system prompt used to generate the output.
+            system_prompt: nil,
+            # The user prompt used to generate the output.
+            user_prompt: nil
+          )
+          end
+
+          sig do
+            override.returns(
+              {
+                ground_truth: String,
+                system_prompt: String,
+                user_prompt: String
+              }
+            )
+          end
+          def to_hash
+          end
+        end
+
+        # Run mode for the evaluation. The run mode allows the user to optimize for speed,
+        # accuracy, and cost by determining which models are used to evaluate the event.
+        module RunMode
+          extend Deeprails::Internal::Type::Enum
+
+          TaggedSymbol =
+            T.type_alias do
+              T.all(
+                Symbol,
+                Deeprails::MonitorDetailResponse::Evaluation::RunMode
+              )
+            end
+          OrSymbol = T.type_alias { T.any(Symbol, String) }
+
+          PRECISION_PLUS =
+            T.let(
+              :precision_plus,
+              Deeprails::MonitorDetailResponse::Evaluation::RunMode::TaggedSymbol
+            )
+          PRECISION =
+            T.let(
+              :precision,
+              Deeprails::MonitorDetailResponse::Evaluation::RunMode::TaggedSymbol
+            )
+          SMART =
+            T.let(
+              :smart,
+              Deeprails::MonitorDetailResponse::Evaluation::RunMode::TaggedSymbol
+            )
+          ECONOMY =
+            T.let(
+              :economy,
+              Deeprails::MonitorDetailResponse::Evaluation::RunMode::TaggedSymbol
+            )
+
+          sig do
+            override.returns(
+              T::Array[
+                Deeprails::MonitorDetailResponse::Evaluation::RunMode::TaggedSymbol
+              ]
+            )
+          end
+          def self.values
+          end
+        end
+
+        module GuardrailMetric
+          extend Deeprails::Internal::Type::Enum
+
+          TaggedSymbol =
+            T.type_alias do
+              T.all(
+                Symbol,
+                Deeprails::MonitorDetailResponse::Evaluation::GuardrailMetric
+              )
+            end
+          OrSymbol = T.type_alias { T.any(Symbol, String) }
+
+          CORRECTNESS =
+            T.let(
+              :correctness,
+              Deeprails::MonitorDetailResponse::Evaluation::GuardrailMetric::TaggedSymbol
+            )
+          COMPLETENESS =
+            T.let(
+              :completeness,
+              Deeprails::MonitorDetailResponse::Evaluation::GuardrailMetric::TaggedSymbol
+            )
+          INSTRUCTION_ADHERENCE =
+            T.let(
+              :instruction_adherence,
+              Deeprails::MonitorDetailResponse::Evaluation::GuardrailMetric::TaggedSymbol
+            )
+          CONTEXT_ADHERENCE =
+            T.let(
+              :context_adherence,
+              Deeprails::MonitorDetailResponse::Evaluation::GuardrailMetric::TaggedSymbol
+            )
+          GROUND_TRUTH_ADHERENCE =
+            T.let(
+              :ground_truth_adherence,
+              Deeprails::MonitorDetailResponse::Evaluation::GuardrailMetric::TaggedSymbol
+            )
+          COMPREHENSIVE_SAFETY =
+            T.let(
+              :comprehensive_safety,
+              Deeprails::MonitorDetailResponse::Evaluation::GuardrailMetric::TaggedSymbol
+            )
+
+          sig do
+            override.returns(
+              T::Array[
+                Deeprails::MonitorDetailResponse::Evaluation::GuardrailMetric::TaggedSymbol
+              ]
+            )
+          end
+          def self.values
+          end
+        end
+      end
+
       class Stats < Deeprails::Internal::Type::BaseModel
         OrHash =
           T.type_alias do

data/rbi/deeprails/models.rbi

@@ -13,11 +13,9 @@ module Deeprails
 
   DefendUpdateWorkflowParams = Deeprails::Models::DefendUpdateWorkflowParams
 
-  EvaluateCreateParams = Deeprails::Models::EvaluateCreateParams
+  FileResponse = Deeprails::Models::FileResponse
 
-  EvaluateRetrieveParams = Deeprails::Models::EvaluateRetrieveParams
-
-  Evaluation = Deeprails::Models::Evaluation
+  FileUploadParams = Deeprails::Models::FileUploadParams
 
   MonitorCreateParams = Deeprails::Models::MonitorCreateParams
 

data/rbi/deeprails/resources/defend.rbi

@@ -18,7 +18,9 @@ module Deeprails
             ],
           custom_hallucination_threshold_values: T::Hash[Symbol, Float],
           description: String,
+          file_search: T::Array[String],
           max_improvement_attempts: Integer,
+          web_search: T::Boolean,
           request_options: Deeprails::RequestOptions::OrHash
         ).returns(Deeprails::DefendResponse)
       end
@@ -48,9 +50,14 @@ module Deeprails
         custom_hallucination_threshold_values: nil,
         # Description for the workflow.
         description: nil,
+        # An array of file IDs to search in the workflow's evaluations. Files must be
+        # uploaded via the DeepRails API first.
+        file_search: nil,
         # Max. number of improvement action retries until a given event passes the
         # guardrails. Defaults to 10.
         max_improvement_attempts: nil,
+        # Whether to enable web search for this workflow's evaluations. Defaults to false.
+        web_search: nil,
         request_options: {}
       )
       end

data/rbi/deeprails/resources/files.rbi

@@ -0,0 +1,26 @@
+# typed: strong
+
+module Deeprails
+  module Resources
+    class Files
+      # Use this endpoint to upload a file to the DeepRails API
+      sig do
+        params(
+          file: Deeprails::Internal::FileInput,
+          request_options: Deeprails::RequestOptions::OrHash
+        ).returns(Deeprails::FileResponse)
+      end
+      def upload(
+        # The contents of the file to upload.
+        file:,
+        request_options: {}
+      )
+      end
+
+      # @api private
+      sig { params(client: Deeprails::Client).returns(T.attached_class) }
+      def self.new(client:)
+      end
+    end
+  end
+end