deeprails 0.1.0

data/lib/deeprails/models/evaluate_create_params.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Deeprails
+  module Models
+    # @see Deeprails::Resources::Evaluate#create
+    class EvaluateCreateParams < Deeprails::Internal::Type::BaseModel
+      extend Deeprails::Internal::Type::RequestParameters::Converter
+      include Deeprails::Internal::Type::RequestParameters
+
+      # @!attribute model_input
+      #   A dictionary of inputs sent to the LLM to generate output. This must contain a
+      #   `user_prompt` field and an optional `context` field. Additional properties are
+      #   allowed.
+      #
+      #   @return [Deeprails::Models::EvaluateCreateParams::ModelInput]
+      required :model_input, -> { Deeprails::EvaluateCreateParams::ModelInput }
+
+      # @!attribute model_output
+      #   Output generated by the LLM to be evaluated.
+      #
+      #   @return [String]
+      required :model_output, String
+
+      # @!attribute run_mode
+      #   Run mode for the evaluation. The run mode allows the user to optimize for speed,
+      #   accuracy, and cost by determining which models are used to evaluate the event.
+      #   Available run modes include `precision_plus`, `precision`, `smart`, and
+      #   `economy`. Defaults to `smart`.
+      #
+      #   @return [Symbol, Deeprails::Models::EvaluateCreateParams::RunMode]
+      required :run_mode, enum: -> { Deeprails::EvaluateCreateParams::RunMode }
+
+      # @!attribute guardrail_metrics
+      #   An array of guardrail metrics that the model input and output pair will be
+      #   evaluated on. For non-enterprise users, these will be limited to the allowed
+      #   guardrail metrics.
+      #
+      #   @return [Array<Symbol, Deeprails::Models::EvaluateCreateParams::GuardrailMetric>, nil]
+      optional :guardrail_metrics,
+               -> { Deeprails::Internal::Type::ArrayOf[enum: Deeprails::EvaluateCreateParams::GuardrailMetric] }
+
+      # @!attribute model_used
+      #   Model ID used to generate the output, like `gpt-4o` or `o3`.
+      #
+      #   @return [String, nil]
+      optional :model_used, String
+
+      # @!attribute nametag
+      #   An optional, user-defined tag for the evaluation.
+      #
+      #   @return [String, nil]
+      optional :nametag, String
+
+      # @!method initialize(model_input:, model_output:, run_mode:, guardrail_metrics: nil, model_used: nil, nametag: nil, request_options: {})
+      #   Some parameter documentations has been truncated, see
+      #   {Deeprails::Models::EvaluateCreateParams} for more details.
+      #
+      #   @param model_input [Deeprails::Models::EvaluateCreateParams::ModelInput] A dictionary of inputs sent to the LLM to generate output. This must contain a
+      #
+      #   @param model_output [String] Output generated by the LLM to be evaluated.
+      #
+      #   @param run_mode [Symbol, Deeprails::Models::EvaluateCreateParams::RunMode] Run mode for the evaluation. The run mode allows the user to optimize for speed
+      #
+      #   @param guardrail_metrics [Array<Symbol, Deeprails::Models::EvaluateCreateParams::GuardrailMetric>] An array of guardrail metrics that the model input and output pair will be evalu
+      #
+      #   @param model_used [String] Model ID used to generate the output, like `gpt-4o` or `o3`.
+      #
+      #   @param nametag [String] An optional, user-defined tag for the evaluation.
+      #
+      #   @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
+
+      class ModelInput < Deeprails::Internal::Type::BaseModel
+        # @!attribute user_prompt
+        #
+        #   @return [String]
+        required :user_prompt, String
+
+        # @!attribute context
+        #
+        #   @return [String, nil]
+        optional :context, String
+
+        # @!method initialize(user_prompt:, context: nil)
+        #   A dictionary of inputs sent to the LLM to generate output. This must contain a
+        #   `user_prompt` field and an optional `context` field. Additional properties are
+        #   allowed.
+        #
+        #   @param user_prompt [String]
+        #   @param context [String]
+      end
+
+      # Run mode for the evaluation. The run mode allows the user to optimize for speed,
+      # accuracy, and cost by determining which models are used to evaluate the event.
+      # Available run modes include `precision_plus`, `precision`, `smart`, and
+      # `economy`. Defaults to `smart`.
+      module RunMode
+        extend Deeprails::Internal::Type::Enum
+
+        PRECISION_PLUS = :precision_plus
+        PRECISION = :precision
+        SMART = :smart
+        ECONOMY = :economy
+
+        # @!method self.values
+        #   @return [Array<Symbol>]
+      end
+
+      module GuardrailMetric
+        extend Deeprails::Internal::Type::Enum
+
+        CORRECTNESS = :correctness
+        COMPLETENESS = :completeness
+        INSTRUCTION_ADHERENCE = :instruction_adherence
+        CONTEXT_ADHERENCE = :context_adherence
+        GROUND_TRUTH_ADHERENCE = :ground_truth_adherence
+        COMPREHENSIVE_SAFETY = :comprehensive_safety
+
+        # @!method self.values
+        #   @return [Array<Symbol>]
+      end
+    end
+  end
+end

data/lib/deeprails/models/evaluate_retrieve_params.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Deeprails
+  module Models
+    # @see Deeprails::Resources::Evaluate#retrieve
+    class EvaluateRetrieveParams < Deeprails::Internal::Type::BaseModel
+      extend Deeprails::Internal::Type::RequestParameters::Converter
+      include Deeprails::Internal::Type::RequestParameters
+
+      # @!method initialize(request_options: {})
+      #   @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
+    end
+  end
+end

data/lib/deeprails/models/evaluation.rb

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+
+module Deeprails
+  module Models
+    # @see Deeprails::Resources::Evaluate#create
+    class Evaluation < Deeprails::Internal::Type::BaseModel
+      # @!attribute eval_id
+      #   A unique evaluation ID.
+      #
+      #   @return [String]
+      required :eval_id, String
+
+      # @!attribute evaluation_status
+      #   Status of the evaluation.
+      #
+      #   @return [Symbol, Deeprails::Models::Evaluation::EvaluationStatus]
+      required :evaluation_status, enum: -> { Deeprails::Evaluation::EvaluationStatus }
+
+      # @!attribute model_input
+      #   A dictionary of inputs sent to the LLM to generate output. The dictionary must
+      #   contain a `user_prompt` field and an optional `context` field. Additional
+      #   properties are allowed.
+      #
+      #   @return [Deeprails::Models::Evaluation::ModelInput]
+      required :model_input, -> { Deeprails::Evaluation::ModelInput }
+
+      # @!attribute model_output
+      #   Output generated by the LLM to be evaluated.
+      #
+      #   @return [String]
+      required :model_output, String
+
+      # @!attribute run_mode
+      #   Run mode for the evaluation. The run mode allows the user to optimize for speed,
+      #   accuracy, and cost by determining which models are used to evaluate the event.
+      #
+      #   @return [Symbol, Deeprails::Models::Evaluation::RunMode]
+      required :run_mode, enum: -> { Deeprails::Evaluation::RunMode }
+
+      # @!attribute created_at
+      #   The time the evaluation was created in UTC.
+      #
+      #   @return [Time, nil]
+      optional :created_at, Time
+
+      # @!attribute end_timestamp
+      #   The time the evaluation completed in UTC.
+      #
+      #   @return [Time, nil]
+      optional :end_timestamp, Time
+
+      # @!attribute error_message
+      #   Description of the error causing the evaluation to fail, if any.
+      #
+      #   @return [String, nil]
+      optional :error_message, String
+
+      # @!attribute error_timestamp
+      #   The time the error causing the evaluation to fail was recorded.
+      #
+      #   @return [Time, nil]
+      optional :error_timestamp, Time
+
+      # @!attribute evaluation_result
+      #   Evaluation result consisting of average scores and rationales for each of the
+      #   evaluated guardrail metrics.
+      #
+      #   @return [Hash{Symbol=>Object}, nil]
+      optional :evaluation_result, Deeprails::Internal::Type::HashOf[Deeprails::Internal::Type::Unknown]
+
+      # @!attribute evaluation_total_cost
+      #   Total cost of the evaluation.
+      #
+      #   @return [Float, nil]
+      optional :evaluation_total_cost, Float
+
+      # @!attribute guardrail_metrics
+      #   An array of guardrail metrics that the model input and output pair will be
+      #   evaluated on.
+      #
+      #   @return [Array<Symbol, Deeprails::Models::Evaluation::GuardrailMetric>, nil]
+      optional :guardrail_metrics,
+               -> { Deeprails::Internal::Type::ArrayOf[enum: Deeprails::Evaluation::GuardrailMetric] }
+
+      # @!attribute model_used
+      #   Model ID used to generate the output, like `gpt-4o` or `o3`.
+      #
+      #   @return [String, nil]
+      optional :model_used, String
+
+      # @!attribute modified_at
+      #   The most recent time the evaluation was modified in UTC.
+      #
+      #   @return [Time, nil]
+      optional :modified_at, Time
+
+      # @!attribute nametag
+      #   An optional, user-defined tag for the evaluation.
+      #
+      #   @return [String, nil]
+      optional :nametag, String
+
+      # @!attribute progress
+      #   Evaluation progress. Values range between 0 and 100; 100 corresponds to a
+      #   completed `evaluation_status`.
+      #
+      #   @return [Integer, nil]
+      optional :progress, Integer
+
+      # @!attribute start_timestamp
+      #   The time the evaluation started in UTC.
+      #
+      #   @return [Time, nil]
+      optional :start_timestamp, Time
+
+      # @!method initialize(eval_id:, evaluation_status:, model_input:, model_output:, run_mode:, created_at: nil, end_timestamp: nil, error_message: nil, error_timestamp: nil, evaluation_result: nil, evaluation_total_cost: nil, guardrail_metrics: nil, model_used: nil, modified_at: nil, nametag: nil, progress: nil, start_timestamp: nil)
+      #   Some parameter documentations has been truncated, see
+      #   {Deeprails::Models::Evaluation} for more details.
+      #
+      #   @param eval_id [String] A unique evaluation ID.
+      #
+      #   @param evaluation_status [Symbol, Deeprails::Models::Evaluation::EvaluationStatus] Status of the evaluation.
+      #
+      #   @param model_input [Deeprails::Models::Evaluation::ModelInput] A dictionary of inputs sent to the LLM to generate output. The dictionary must c
+      #
+      #   @param model_output [String] Output generated by the LLM to be evaluated.
+      #
+      #   @param run_mode [Symbol, Deeprails::Models::Evaluation::RunMode] Run mode for the evaluation. The run mode allows the user to optimize for speed
+      #
+      #   @param created_at [Time] The time the evaluation was created in UTC.
+      #
+      #   @param end_timestamp [Time] The time the evaluation completed in UTC.
+      #
+      #   @param error_message [String] Description of the error causing the evaluation to fail, if any.
+      #
+      #   @param error_timestamp [Time] The time the error causing the evaluation to fail was recorded.
+      #
+      #   @param evaluation_result [Hash{Symbol=>Object}] Evaluation result consisting of average scores and rationales for each of the ev
+      #
+      #   @param evaluation_total_cost [Float] Total cost of the evaluation.
+      #
+      #   @param guardrail_metrics [Array<Symbol, Deeprails::Models::Evaluation::GuardrailMetric>] An array of guardrail metrics that the model input and output pair will be evalu
+      #
+      #   @param model_used [String] Model ID used to generate the output, like `gpt-4o` or `o3`.
+      #
+      #   @param modified_at [Time] The most recent time the evaluation was modified in UTC.
+      #
+      #   @param nametag [String] An optional, user-defined tag for the evaluation.
+      #
+      #   @param progress [Integer] Evaluation progress. Values range between 0 and 100; 100 corresponds to a compl
+      #
+      #   @param start_timestamp [Time] The time the evaluation started in UTC.
+
+      # Status of the evaluation.
+      #
+      # @see Deeprails::Models::Evaluation#evaluation_status
+      module EvaluationStatus
+        extend Deeprails::Internal::Type::Enum
+
+        IN_PROGRESS = :in_progress
+        COMPLETED = :completed
+        CANCELED = :canceled
+        QUEUED = :queued
+        FAILED = :failed
+
+        # @!method self.values
+        #   @return [Array<Symbol>]
+      end
+
+      # @see Deeprails::Models::Evaluation#model_input
+      class ModelInput < Deeprails::Internal::Type::BaseModel
+        # @!attribute user_prompt
+        #   The user prompt used to generate the output.
+        #
+        #   @return [String]
+        required :user_prompt, String
+
+        # @!attribute context
+        #   Optional context supplied to the LLM when generating the output.
+        #
+        #   @return [String, nil]
+        optional :context, String
+
+        # @!method initialize(user_prompt:, context: nil)
+        #   A dictionary of inputs sent to the LLM to generate output. The dictionary must
+        #   contain a `user_prompt` field and an optional `context` field. Additional
+        #   properties are allowed.
+        #
+        #   @param user_prompt [String] The user prompt used to generate the output.
+        #
+        #   @param context [String] Optional context supplied to the LLM when generating the output.
+      end
+
+      # Run mode for the evaluation. The run mode allows the user to optimize for speed,
+      # accuracy, and cost by determining which models are used to evaluate the event.
+      #
+      # @see Deeprails::Models::Evaluation#run_mode
+      module RunMode
+        extend Deeprails::Internal::Type::Enum
+
+        PRECISION_PLUS = :precision_plus
+        PRECISION = :precision
+        SMART = :smart
+        ECONOMY = :economy
+
+        # @!method self.values
+        #   @return [Array<Symbol>]
+      end
+
+      module GuardrailMetric
+        extend Deeprails::Internal::Type::Enum
+
+        CORRECTNESS = :correctness
+        COMPLETENESS = :completeness
+        INSTRUCTION_ADHERENCE = :instruction_adherence
+        CONTEXT_ADHERENCE = :context_adherence
+        GROUND_TRUTH_ADHERENCE = :ground_truth_adherence
+        COMPREHENSIVE_SAFETY = :comprehensive_safety
+
+        # @!method self.values
+        #   @return [Array<Symbol>]
+      end
+    end
+  end
+end

data/lib/deeprails/models/monitor_create_params.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Deeprails
+  module Models
+    # @see Deeprails::Resources::Monitor#create
+    class MonitorCreateParams < Deeprails::Internal::Type::BaseModel
+      extend Deeprails::Internal::Type::RequestParameters::Converter
+      include Deeprails::Internal::Type::RequestParameters
+
+      # @!attribute name
+      #   Name of the new monitor.
+      #
+      #   @return [String]
+      required :name, String
+
+      # @!attribute description
+      #   Description of the new monitor.
+      #
+      #   @return [String, nil]
+      optional :description, String
+
+      # @!method initialize(name:, description: nil, request_options: {})
+      #   @param name [String] Name of the new monitor.
+      #
+      #   @param description [String] Description of the new monitor.
+      #
+      #   @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
+    end
+  end
+end

data/lib/deeprails/models/monitor_retrieve_params.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Deeprails
+  module Models
+    # @see Deeprails::Resources::Monitor#retrieve
+    class MonitorRetrieveParams < Deeprails::Internal::Type::BaseModel
+      extend Deeprails::Internal::Type::RequestParameters::Converter
+      include Deeprails::Internal::Type::RequestParameters
+
+      # @!attribute limit
+      #   Limit the returned events associated with this monitor. Defaults to 10.
+      #
+      #   @return [Integer, nil]
+      optional :limit, Integer
+
+      # @!method initialize(limit: nil, request_options: {})
+      #   @param limit [Integer] Limit the returned events associated with this monitor. Defaults to 10.
+      #
+      #   @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
+    end
+  end
+end

data/lib/deeprails/models/monitor_retrieve_response.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+module Deeprails
+  module Models
+    # @see Deeprails::Resources::Monitor#retrieve
+    class MonitorRetrieveResponse < Deeprails::Internal::Type::BaseModel
+      # @!attribute success
+      #   Represents whether the request was completed successfully.
+      #
+      #   @return [Boolean]
+      required :success, Deeprails::Internal::Type::Boolean
+
+      # @!attribute data
+      #   Detailed response payload for retrieving a monitor and its evaluations.
+      #
+      #   @return [Deeprails::Models::MonitorRetrieveResponse::Data, nil]
+      optional :data, -> { Deeprails::Models::MonitorRetrieveResponse::Data }
+
+      # @!attribute message
+      #   The accompanying message for the request. Includes error details when
+      #   applicable.
+      #
+      #   @return [String, nil]
+      optional :message, String
+
+      # @!method initialize(success:, data: nil, message: nil)
+      #   Some parameter documentations has been truncated, see
+      #   {Deeprails::Models::MonitorRetrieveResponse} for more details.
+      #
+      #   Response wrapper for operations returning a MonitorDetailResponse.
+      #
+      #   @param success [Boolean] Represents whether the request was completed successfully.
+      #
+      #   @param data [Deeprails::Models::MonitorRetrieveResponse::Data] Detailed response payload for retrieving a monitor and its evaluations.
+      #
+      #   @param message [String] The accompanying message for the request. Includes error details when applicable
+
+      # @see Deeprails::Models::MonitorRetrieveResponse#data
+      class Data < Deeprails::Internal::Type::BaseModel
+        # @!attribute monitor_id
+        #   A unique monitor ID.
+        #
+        #   @return [String]
+        required :monitor_id, String
+
+        # @!attribute monitor_status
+        #   Status of the monitor. Can be `active` or `inactive`. Inactive monitors no
+        #   longer record and evaluate events.
+        #
+        #   @return [Symbol, Deeprails::Models::MonitorRetrieveResponse::Data::MonitorStatus]
+        required :monitor_status, enum: -> { Deeprails::Models::MonitorRetrieveResponse::Data::MonitorStatus }
+
+        # @!attribute name
+        #   Name of this monitor.
+        #
+        #   @return [String]
+        required :name, String
+
+        # @!attribute created_at
+        #   The time the monitor was created in UTC.
+        #
+        #   @return [Time, nil]
+        optional :created_at, Time
+
+        # @!attribute description
+        #   Description of this monitor.
+        #
+        #   @return [String, nil]
+        optional :description, String
+
+        # @!attribute evaluations
+        #   An array of all evaluations performed by this monitor. Each one corresponds to a
+        #   separate monitor event.
+        #
+        #   @return [Array<Deeprails::Models::Evaluation>, nil]
+        optional :evaluations, -> { Deeprails::Internal::Type::ArrayOf[Deeprails::Evaluation] }
+
+        # @!attribute stats
+        #   Contains five fields used for stats of this monitor: total evaluations,
+        #   completed evaluations, failed evaluations, queued evaluations, and in progress
+        #   evaluations.
+        #
+        #   @return [Deeprails::Models::MonitorRetrieveResponse::Data::Stats, nil]
+        optional :stats, -> { Deeprails::Models::MonitorRetrieveResponse::Data::Stats }
+
+        # @!attribute updated_at
+        #   The most recent time the monitor was modified in UTC.
+        #
+        #   @return [Time, nil]
+        optional :updated_at, Time
+
+        # @!attribute user_id
+        #   User ID of the user who created the monitor.
+        #
+        #   @return [String, nil]
+        optional :user_id, String
+
+        # @!method initialize(monitor_id:, monitor_status:, name:, created_at: nil, description: nil, evaluations: nil, stats: nil, updated_at: nil, user_id: nil)
+        #   Some parameter documentations has been truncated, see
+        #   {Deeprails::Models::MonitorRetrieveResponse::Data} for more details.
+        #
+        #   Detailed response payload for retrieving a monitor and its evaluations.
+        #
+        #   @param monitor_id [String] A unique monitor ID.
+        #
+        #   @param monitor_status [Symbol, Deeprails::Models::MonitorRetrieveResponse::Data::MonitorStatus] Status of the monitor. Can be `active` or `inactive`. Inactive monitors no lon
+        #
+        #   @param name [String] Name of this monitor.
+        #
+        #   @param created_at [Time] The time the monitor was created in UTC.
+        #
+        #   @param description [String] Description of this monitor.
+        #
+        #   @param evaluations [Array<Deeprails::Models::Evaluation>] An array of all evaluations performed by this monitor. Each one corresponds to
+        #
+        #   @param stats [Deeprails::Models::MonitorRetrieveResponse::Data::Stats] Contains five fields used for stats of this monitor: total evaluations, complete
+        #
+        #   @param updated_at [Time] The most recent time the monitor was modified in UTC.
+        #
+        #   @param user_id [String] User ID of the user who created the monitor.
+
+        # Status of the monitor. Can be `active` or `inactive`. Inactive monitors no
+        # longer record and evaluate events.
+        #
+        # @see Deeprails::Models::MonitorRetrieveResponse::Data#monitor_status
+        module MonitorStatus
+          extend Deeprails::Internal::Type::Enum
+
+          ACTIVE = :active
+          INACTIVE = :inactive
+
+          # @!method self.values
+          #   @return [Array<Symbol>]
+        end
+
+        # @see Deeprails::Models::MonitorRetrieveResponse::Data#stats
+        class Stats < Deeprails::Internal::Type::BaseModel
+          # @!attribute completed_evaluations
+          #   Number of evaluations that completed successfully.
+          #
+          #   @return [Integer, nil]
+          optional :completed_evaluations, Integer
+
+          # @!attribute failed_evaluations
+          #   Number of evaluations that failed.
+          #
+          #   @return [Integer, nil]
+          optional :failed_evaluations, Integer
+
+          # @!attribute in_progress_evaluations
+          #   Number of evaluations currently in progress.
+          #
+          #   @return [Integer, nil]
+          optional :in_progress_evaluations, Integer
+
+          # @!attribute queued_evaluations
+          #   Number of evaluations currently queued.
+          #
+          #   @return [Integer, nil]
+          optional :queued_evaluations, Integer
+
+          # @!attribute total_evaluations
+          #   Total number of evaluations performed by this monitor.
+          #
+          #   @return [Integer, nil]
+          optional :total_evaluations, Integer
+
+          # @!method initialize(completed_evaluations: nil, failed_evaluations: nil, in_progress_evaluations: nil, queued_evaluations: nil, total_evaluations: nil)
+          #   Contains five fields used for stats of this monitor: total evaluations,
+          #   completed evaluations, failed evaluations, queued evaluations, and in progress
+          #   evaluations.
+          #
+          #   @param completed_evaluations [Integer] Number of evaluations that completed successfully.
+          #
+          #   @param failed_evaluations [Integer] Number of evaluations that failed.
+          #
+          #   @param in_progress_evaluations [Integer] Number of evaluations currently in progress.
+          #
+          #   @param queued_evaluations [Integer] Number of evaluations currently queued.
+          #
+          #   @param total_evaluations [Integer] Total number of evaluations performed by this monitor.
+        end
+      end
+    end
+  end
+end