deeprails 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4059b07f012b97e31dfe283afca63de3e15fbcd8b4707a83b1a89bb2c064ac73
-  data.tar.gz: a15f42872e3ea1ed96fa9fafc507f20725532dc9a65e09f573863d9030b38879
+  metadata.gz: 0db85ba4c5a974e1efaa1db27b1888cbbf8ae21c5b1d15b4d75733f662c42a8a
+  data.tar.gz: 1ff3fc484f0032e6dddae533e803d7f591d270527428abba892879e5110105d3
 SHA512:
-  metadata.gz: 1df27d336394dbfa922d7ba6e410b7d37918803fb3e0c85706774bac03cdad2ebc48491a8c988306129e72f66b87054866d7d76d0a760312f0ba4b5c84d93a4e
-  data.tar.gz: a07104cbb7fec46daa6cd991194c370aea807ce7d4e2732c0dd26eb485952ff4962b33073ff2411769612bdb9661ab2d911105a090766ae954d8a802620d1f18
+  metadata.gz: 12ba483e35868225c32653c0f8d3d464944d29e76633f52616e2557a613355222f886caae2aa3c908d535d5e693c92349ec5a8ebe5404c82ecd8d156ade46a96
+  data.tar.gz: 68678894ea182b88ad6dd35b1cbedf1c2f86c210a7769180f2ac82d71f2687a1c96ecd8ca032e0541acd38bee83e0447cbb4fd5ff09f50d9c6fd8f8ea163ec0b

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 0.9.0 (2025-10-29)
+
+Full Changelog: [v0.8.0...v0.9.0](https://github.com/deeprails/deeprails-ruby-sdk/compare/v0.8.0...v0.9.0)
+
+### Features
+
+* **api:** remove evaluate api ([733236c](https://github.com/deeprails/deeprails-ruby-sdk/commit/733236c71a7bad030caea5ebe42a3c8061e05ec6))
+* **api:** remove evaluate references ([de622dd](https://github.com/deeprails/deeprails-ruby-sdk/commit/de622dd89f3c14f5db366bd425c1e4c68bc59886))
+
 ## 0.8.0 (2025-10-24)
 
 Full Changelog: [v0.7.0...v0.8.0](https://github.com/deeprails/deeprails-ruby-sdk/compare/v0.7.0...v0.8.0)

data/README.md

@@ -15,7 +15,7 @@ To use this gem, install via Bundler by adding the following to your application
 <!-- x-release-please-start-version -->
 
 ```ruby
-gem "deeprails", "~> 0.8.0"
+gem "deeprails", "~> 0.9.0"
 ```
 
 <!-- x-release-please-end -->

data/lib/deeprails/client.rb

@@ -24,9 +24,6 @@ module Deeprails
     # @return [Deeprails::Resources::Monitor]
     attr_reader :monitor
 
-    # @return [Deeprails::Resources::Evaluate]
-    attr_reader :evaluate
-
     # @api private
     #
     # @return [Hash{String=>String}]
@@ -76,7 +73,6 @@ module Deeprails
 
       @defend = Deeprails::Resources::Defend.new(client: self)
       @monitor = Deeprails::Resources::Monitor.new(client: self)
-      @evaluate = Deeprails::Resources::Evaluate.new(client: self)
     end
   end
 end

data/lib/deeprails/models/monitor_detail_response.rb

@@ -39,8 +39,9 @@ module Deeprails
       #   An array of all evaluations performed by this monitor. Each one corresponds to a
       #   separate monitor event.
       #
-      #   @return [Array<Deeprails::Models::Evaluation>, nil]
-      optional :evaluations, -> { Deeprails::Internal::Type::ArrayOf[Deeprails::Evaluation] }
+      #   @return [Array<Deeprails::Models::MonitorDetailResponse::Evaluation>, nil]
+      optional :evaluations,
+               -> { Deeprails::Internal::Type::ArrayOf[Deeprails::MonitorDetailResponse::Evaluation] }
 
       # @!attribute stats
       #   Contains five fields used for stats of this monitor: total evaluations,
@@ -76,7 +77,7 @@ module Deeprails
       #
       #   @param description [String] Description of this monitor.
       #
-      #   @param evaluations [Array<Deeprails::Models::Evaluation>] An array of all evaluations performed by this monitor. Each one corresponds to
+      #   @param evaluations [Array<Deeprails::Models::MonitorDetailResponse::Evaluation>] An array of all evaluations performed by this monitor. Each one corresponds to
       #
       #   @param stats [Deeprails::Models::MonitorDetailResponse::Stats] Contains five fields used for stats of this monitor: total evaluations, complete
       #
@@ -98,6 +99,233 @@ module Deeprails
         #   @return [Array<Symbol>]
       end
 
+      class Evaluation < Deeprails::Internal::Type::BaseModel
+        # @!attribute eval_id
+        #   A unique evaluation ID.
+        #
+        #   @return [String]
+        required :eval_id, String
+
+        # @!attribute evaluation_status
+        #   Status of the evaluation.
+        #
+        #   @return [Symbol, Deeprails::Models::MonitorDetailResponse::Evaluation::EvaluationStatus]
+        required :evaluation_status, enum: -> { Deeprails::MonitorDetailResponse::Evaluation::EvaluationStatus }
+
+        # @!attribute model_input
+        #   A dictionary of inputs sent to the LLM to generate output. The dictionary must
+        #   contain at least a `user_prompt` field or a `system_prompt` field. For
+        #   ground_truth_adherence guardrail metric, `ground_truth` should be provided.
+        #
+        #   @return [Deeprails::Models::MonitorDetailResponse::Evaluation::ModelInput]
+        required :model_input, -> { Deeprails::MonitorDetailResponse::Evaluation::ModelInput }
+
+        # @!attribute model_output
+        #   Output generated by the LLM to be evaluated.
+        #
+        #   @return [String]
+        required :model_output, String
+
+        # @!attribute run_mode
+        #   Run mode for the evaluation. The run mode allows the user to optimize for speed,
+        #   accuracy, and cost by determining which models are used to evaluate the event.
+        #
+        #   @return [Symbol, Deeprails::Models::MonitorDetailResponse::Evaluation::RunMode]
+        required :run_mode, enum: -> { Deeprails::MonitorDetailResponse::Evaluation::RunMode }
+
+        # @!attribute created_at
+        #   The time the evaluation was created in UTC.
+        #
+        #   @return [Time, nil]
+        optional :created_at, Time
+
+        # @!attribute end_timestamp
+        #   The time the evaluation completed in UTC.
+        #
+        #   @return [Time, nil]
+        optional :end_timestamp, Time
+
+        # @!attribute error_message
+        #   Description of the error causing the evaluation to fail, if any.
+        #
+        #   @return [String, nil]
+        optional :error_message, String
+
+        # @!attribute error_timestamp
+        #   The time the error causing the evaluation to fail was recorded.
+        #
+        #   @return [Time, nil]
+        optional :error_timestamp, Time
+
+        # @!attribute evaluation_result
+        #   Evaluation result consisting of average scores and rationales for each of the
+        #   evaluated guardrail metrics.
+        #
+        #   @return [Hash{Symbol=>Object}, nil]
+        optional :evaluation_result, Deeprails::Internal::Type::HashOf[Deeprails::Internal::Type::Unknown]
+
+        # @!attribute evaluation_total_cost
+        #   Total cost of the evaluation.
+        #
+        #   @return [Float, nil]
+        optional :evaluation_total_cost, Float
+
+        # @!attribute guardrail_metrics
+        #   An array of guardrail metrics that the model input and output pair will be
+        #   evaluated on.
+        #
+        #   @return [Array<Symbol, Deeprails::Models::MonitorDetailResponse::Evaluation::GuardrailMetric>, nil]
+        optional :guardrail_metrics,
+                 -> { Deeprails::Internal::Type::ArrayOf[enum: Deeprails::MonitorDetailResponse::Evaluation::GuardrailMetric] }
+
+        # @!attribute model_used
+        #   Model ID used to generate the output, like `gpt-4o` or `o3`.
+        #
+        #   @return [String, nil]
+        optional :model_used, String
+
+        # @!attribute modified_at
+        #   The most recent time the evaluation was modified in UTC.
+        #
+        #   @return [Time, nil]
+        optional :modified_at, Time
+
+        # @!attribute nametag
+        #   An optional, user-defined tag for the evaluation.
+        #
+        #   @return [String, nil]
+        optional :nametag, String
+
+        # @!attribute progress
+        #   Evaluation progress. Values range between 0 and 100; 100 corresponds to a
+        #   completed `evaluation_status`.
+        #
+        #   @return [Integer, nil]
+        optional :progress, Integer
+
+        # @!attribute start_timestamp
+        #   The time the evaluation started in UTC.
+        #
+        #   @return [Time, nil]
+        optional :start_timestamp, Time
+
+        # @!method initialize(eval_id:, evaluation_status:, model_input:, model_output:, run_mode:, created_at: nil, end_timestamp: nil, error_message: nil, error_timestamp: nil, evaluation_result: nil, evaluation_total_cost: nil, guardrail_metrics: nil, model_used: nil, modified_at: nil, nametag: nil, progress: nil, start_timestamp: nil)
+        #   Some parameter documentations has been truncated, see
+        #   {Deeprails::Models::MonitorDetailResponse::Evaluation} for more details.
+        #
+        #   @param eval_id [String] A unique evaluation ID.
+        #
+        #   @param evaluation_status [Symbol, Deeprails::Models::MonitorDetailResponse::Evaluation::EvaluationStatus] Status of the evaluation.
+        #
+        #   @param model_input [Deeprails::Models::MonitorDetailResponse::Evaluation::ModelInput] A dictionary of inputs sent to the LLM to generate output. The dictionary must c
+        #
+        #   @param model_output [String] Output generated by the LLM to be evaluated.
+        #
+        #   @param run_mode [Symbol, Deeprails::Models::MonitorDetailResponse::Evaluation::RunMode] Run mode for the evaluation. The run mode allows the user to optimize for speed
+        #
+        #   @param created_at [Time] The time the evaluation was created in UTC.
+        #
+        #   @param end_timestamp [Time] The time the evaluation completed in UTC.
+        #
+        #   @param error_message [String] Description of the error causing the evaluation to fail, if any.
+        #
+        #   @param error_timestamp [Time] The time the error causing the evaluation to fail was recorded.
+        #
+        #   @param evaluation_result [Hash{Symbol=>Object}] Evaluation result consisting of average scores and rationales for each of the ev
+        #
+        #   @param evaluation_total_cost [Float] Total cost of the evaluation.
+        #
+        #   @param guardrail_metrics [Array<Symbol, Deeprails::Models::MonitorDetailResponse::Evaluation::GuardrailMetric>] An array of guardrail metrics that the model input and output pair will be evalu
+        #
+        #   @param model_used [String] Model ID used to generate the output, like `gpt-4o` or `o3`.
+        #
+        #   @param modified_at [Time] The most recent time the evaluation was modified in UTC.
+        #
+        #   @param nametag [String] An optional, user-defined tag for the evaluation.
+        #
+        #   @param progress [Integer] Evaluation progress. Values range between 0 and 100; 100 corresponds to a compl
+        #
+        #   @param start_timestamp [Time] The time the evaluation started in UTC.
+
+        # Status of the evaluation.
+        #
+        # @see Deeprails::Models::MonitorDetailResponse::Evaluation#evaluation_status
+        module EvaluationStatus
+          extend Deeprails::Internal::Type::Enum
+
+          IN_PROGRESS = :in_progress
+          COMPLETED = :completed
+          CANCELED = :canceled
+          QUEUED = :queued
+          FAILED = :failed
+
+          # @!method self.values
+          #   @return [Array<Symbol>]
+        end
+
+        # @see Deeprails::Models::MonitorDetailResponse::Evaluation#model_input
+        class ModelInput < Deeprails::Internal::Type::BaseModel
+          # @!attribute ground_truth
+          #   The ground truth for evaluating Ground Truth Adherence guardrail.
+          #
+          #   @return [String, nil]
+          optional :ground_truth, String
+
+          # @!attribute system_prompt
+          #   The system prompt used to generate the output.
+          #
+          #   @return [String, nil]
+          optional :system_prompt, String
+
+          # @!attribute user_prompt
+          #   The user prompt used to generate the output.
+          #
+          #   @return [String, nil]
+          optional :user_prompt, String
+
+          # @!method initialize(ground_truth: nil, system_prompt: nil, user_prompt: nil)
+          #   A dictionary of inputs sent to the LLM to generate output. The dictionary must
+          #   contain at least a `user_prompt` field or a `system_prompt` field. For
+          #   ground_truth_adherence guardrail metric, `ground_truth` should be provided.
+          #
+          #   @param ground_truth [String] The ground truth for evaluating Ground Truth Adherence guardrail.
+          #
+          #   @param system_prompt [String] The system prompt used to generate the output.
+          #
+          #   @param user_prompt [String] The user prompt used to generate the output.
+        end
+
+        # Run mode for the evaluation. The run mode allows the user to optimize for speed,
+        # accuracy, and cost by determining which models are used to evaluate the event.
+        #
+        # @see Deeprails::Models::MonitorDetailResponse::Evaluation#run_mode
+        module RunMode
+          extend Deeprails::Internal::Type::Enum
+
+          PRECISION_PLUS = :precision_plus
+          PRECISION = :precision
+          SMART = :smart
+          ECONOMY = :economy
+
+          # @!method self.values
+          #   @return [Array<Symbol>]
+        end
+
+        module GuardrailMetric
+          extend Deeprails::Internal::Type::Enum
+
+          CORRECTNESS = :correctness
+          COMPLETENESS = :completeness
+          INSTRUCTION_ADHERENCE = :instruction_adherence
+          CONTEXT_ADHERENCE = :context_adherence
+          GROUND_TRUTH_ADHERENCE = :ground_truth_adherence
+          COMPREHENSIVE_SAFETY = :comprehensive_safety
+
+          # @!method self.values
+          #   @return [Array<Symbol>]
+        end
+      end
+
       # @see Deeprails::Models::MonitorDetailResponse#stats
       class Stats < Deeprails::Internal::Type::BaseModel
         # @!attribute completed_evaluations

data/lib/deeprails/models.rb

@@ -51,12 +51,6 @@ module Deeprails
 
   DefendUpdateWorkflowParams = Deeprails::Models::DefendUpdateWorkflowParams
 
-  EvaluateCreateParams = Deeprails::Models::EvaluateCreateParams
-
-  EvaluateRetrieveParams = Deeprails::Models::EvaluateRetrieveParams
-
-  Evaluation = Deeprails::Models::Evaluation
-
   MonitorCreateParams = Deeprails::Models::MonitorCreateParams
 
   MonitorDetailResponse = Deeprails::Models::MonitorDetailResponse

data/lib/deeprails/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deeprails
-  VERSION = "0.8.0"
+  VERSION = "0.9.0"
 end

data/lib/deeprails.rb

@@ -56,9 +56,6 @@ require_relative "deeprails/models/defend_retrieve_event_params"
 require_relative "deeprails/models/defend_retrieve_workflow_params"
 require_relative "deeprails/models/defend_submit_event_params"
 require_relative "deeprails/models/defend_update_workflow_params"
-require_relative "deeprails/models/evaluate_create_params"
-require_relative "deeprails/models/evaluate_retrieve_params"
-require_relative "deeprails/models/evaluation"
 require_relative "deeprails/models/monitor_create_params"
 require_relative "deeprails/models/monitor_detail_response"
 require_relative "deeprails/models/monitor_event_response"
@@ -69,5 +66,4 @@ require_relative "deeprails/models/monitor_update_params"
 require_relative "deeprails/models/workflow_event_response"
 require_relative "deeprails/models"
 require_relative "deeprails/resources/defend"
-require_relative "deeprails/resources/evaluate"
 require_relative "deeprails/resources/monitor"

data/rbi/deeprails/client.rbi

@@ -19,9 +19,6 @@ module Deeprails
     sig { returns(Deeprails::Resources::Monitor) }
     attr_reader :monitor
 
-    sig { returns(Deeprails::Resources::Evaluate) }
-    attr_reader :evaluate
-
     # @api private
     sig { override.returns(T::Hash[String, String]) }
     private def auth_headers