deeprails 0.18.0 → 0.24.0

data/rbi/deeprails/resources/defend.rbi

@@ -17,6 +17,7 @@ module Deeprails
               Symbol,
               Deeprails::DefendCreateWorkflowParams::AutomaticHallucinationToleranceLevel::OrSymbol
             ],
+          context_awareness: T::Boolean,
           custom_hallucination_threshold_values: T::Hash[Symbol, Float],
           description: String,
           file_search: T::Array[String],
@@ -45,6 +46,13 @@ module Deeprails
         # `instruction_adherence`, `context_adherence`, `ground_truth_adherence`, or
         # `comprehensive_safety`.
         automatic_hallucination_tolerance_levels: nil,
+        # Context includes any structured information that directly relates to the model’s
+        # input and expected output—e.g., the recent turn-by-turn history between an AI
+        # tutor and a student, facts or state passed through an agentic workflow, or other
+        # domain-specific signals your system already knows and wants the model to
+        # condition on. This field determines whether to enable context awareness for this
+        # workflow's evaluations. Defaults to false.
+        context_awareness: nil,
         # Mapping of guardrail metrics to floating point threshold values. Possible
         # metrics are `correctness`, `completeness`, `instruction_adherence`,
         # `context_adherence`, `ground_truth_adherence`, or `comprehensive_safety`.
@@ -98,6 +106,44 @@ module Deeprails
       )
       end
 
+      # Use this endpoint to submit a model input and output pair to a workflow for
+      # evaluation with streaming responses.
+      sig do
+        params(
+          workflow_id: String,
+          model_input: T::Hash[Symbol, T.anything],
+          model_output: String,
+          model_used: String,
+          run_mode:
+            Deeprails::DefendSubmitAndStreamEventParams::RunMode::OrSymbol,
+          stream: T::Boolean,
+          nametag: String,
+          request_options: Deeprails::RequestOptions::OrHash
+        ).returns(Deeprails::Internal::Stream[String])
+      end
+      def submit_and_stream_event_streaming(
+        # Path param: The ID of the workflow to create the event for.
+        workflow_id,
+        # Body param: The input provided to the model (e.g., prompt, messages).
+        model_input:,
+        # Body param: The output generated by the model to be evaluated.
+        model_output:,
+        # Body param: The model that generated the output (e.g., "gpt-4", "claude-3").
+        model_used:,
+        # Body param: The evaluation run mode. Streaming is supported on all run modes
+        # except precision_max and precision_max_codex. Note: super_fast does not support
+        # Web Search or File Search — if your workflow has these enabled, use a different
+        # run mode or disable the capability on the workflow.
+        run_mode:,
+        # Query param: Enable SSE streaming for real-time token feedback. Supported on all
+        # run modes except precision_max and precision_max_codex.
+        stream: nil,
+        # Body param: Optional tag to identify this event.
+        nametag: nil,
+        request_options: {}
+      )
+      end
+
       # Use this endpoint to submit a model input and output pair to a workflow for
       # evaluation
       sig do
@@ -115,8 +161,8 @@ module Deeprails
         # Workflow ID associated with this event.
         workflow_id,
         # A dictionary of inputs sent to the LLM to generate output. The dictionary must
-        # contain at least a `user_prompt` field or a `system_prompt` field. For the
-        # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
+        # contain a `user_prompt` field. For the ground_truth_adherence guardrail metric,
+        # `ground_truth` should be provided.
         model_input:,
         # Output generated by the LLM to be evaluated.
         model_output:,
@@ -124,8 +170,11 @@ module Deeprails
         model_used:,
         # Run mode for the workflow event. The run mode allows the user to optimize for
         # speed, accuracy, and cost by determining which models are used to evaluate the
-        # event. Available run modes include `precision_plus`, `precision`, `smart`, and
-        # `economy`. Defaults to `smart`.
+        # event. Available run modes (fastest to most thorough): `super_fast`, `fast`,
+        # `precision`, `precision_codex`, `precision_max`, and `precision_max_codex`.
+        # Defaults to `fast`. Note: `super_fast` does not support Web Search or File
+        # Search — if your workflow has these capabilities enabled, use a different run
+        # mode or edit the workflow to disable them.
         run_mode:,
         # An optional, user-defined tag for the event.
         nametag: nil,
@@ -137,18 +186,64 @@ module Deeprails
       sig do
         params(
           workflow_id: String,
+          automatic_hallucination_tolerance_levels:
+            T::Hash[
+              Symbol,
+              Deeprails::DefendUpdateWorkflowParams::AutomaticHallucinationToleranceLevel::OrSymbol
+            ],
+          context_awareness: T::Boolean,
+          custom_hallucination_threshold_values: T::Hash[Symbol, Float],
           description: String,
+          file_search: T::Array[String],
+          improvement_action:
+            Deeprails::DefendUpdateWorkflowParams::ImprovementAction::OrSymbol,
+          max_improvement_attempts: Integer,
           name: String,
+          threshold_type:
+            Deeprails::DefendUpdateWorkflowParams::ThresholdType::OrSymbol,
+          web_search: T::Boolean,
           request_options: Deeprails::RequestOptions::OrHash
         ).returns(Deeprails::DefendUpdateResponse)
       end
       def update_workflow(
         # The ID of the workflow to edit.
         workflow_id,
-        # Description for the workflow.
+        # New mapping of guardrail metrics to hallucination tolerance levels (either
+        # `low`, `medium`, or `high`) to be used when `threshold_type` is set to
+        # `automatic`. Possible metrics are `completeness`, `instruction_adherence`,
+        # `context_adherence`, `ground_truth_adherence`, or `comprehensive_safety`.
+        automatic_hallucination_tolerance_levels: nil,
+        # Whether to enable context awareness for this workflow's evaluations.
+        context_awareness: nil,
+        # New mapping of guardrail metrics to floating point threshold values to be used
+        # when `threshold_type` is set to `custom`. Possible metrics are `correctness`,
+        # `completeness`, `instruction_adherence`, `context_adherence`,
+        # `ground_truth_adherence`, or `comprehensive_safety`.
+        custom_hallucination_threshold_values: nil,
+        # New description for the workflow.
         description: nil,
-        # Name of the workflow.
+        # An array of file IDs to search in the workflow's evaluations. Files must be
+        # uploaded via the DeepRails API first.
+        file_search: nil,
+        # The new action used to improve outputs that fail one or more guardrail metrics
+        # for the workflow events. May be `regen`, `fixit`, or `do_nothing`. ReGen runs
+        # the user's input prompt with minor induced variance. FixIt attempts to directly
+        # address the shortcomings of the output using the guardrail failure rationale. Do
+        # Nothing does not attempt any improvement.
+        improvement_action: nil,
+        # Max. number of improvement action attempts until a given event passes the
+        # guardrails. Defaults to 10.
+        max_improvement_attempts: nil,
+        # New name for the workflow.
         name: nil,
+        # New type of thresholds to use for the workflow, either `automatic` or `custom`.
+        # Automatic thresholds are assigned internally after the user specifies a
+        # qualitative tolerance for the metrics, whereas custom metrics allow the user to
+        # set the threshold for each metric as a floating point number between 0.0 and
+        # 1.0.
+        threshold_type: nil,
+        # Whether to enable web search for this workflow's evaluations.
+        web_search: nil,
         request_options: {}
       )
       end

data/rbi/deeprails/resources/monitor.rbi

@@ -10,6 +10,7 @@ module Deeprails
           guardrail_metrics:
             T::Array[Deeprails::MonitorCreateParams::GuardrailMetric::OrSymbol],
           name: String,
+          context_awareness: T::Boolean,
           description: String,
           file_search: T::Array[String],
           web_search: T::Boolean,
@@ -24,6 +25,13 @@ module Deeprails
         guardrail_metrics:,
         # Name of the new monitor.
         name:,
+        # Context includes any structured information that directly relates to the model’s
+        # input and expected output—e.g., the recent turn-by-turn history between an AI
+        # tutor and a student, facts or state passed through an agentic workflow, or other
+        # domain-specific signals your system already knows and wants the model to
+        # condition on. This field determines whether to enable context awareness for this
+        # monitor's evaluations. Defaults to false.
+        context_awareness: nil,
         # Description of the new monitor.
         description: nil,
         # An array of file IDs to search in the monitor's evaluations. Files must be
@@ -54,27 +62,39 @@ module Deeprails
       )
       end
 
-      # Use this endpoint to update the name, description, or status of an existing
-      # monitor
+      # Use this endpoint to update the name, status, and/or other details of an
+      # existing monitor.
       sig do
         params(
           monitor_id: String,
           description: String,
+          file_search: T::Array[String],
+          guardrail_metrics:
+            T::Array[Deeprails::MonitorUpdateParams::GuardrailMetric::OrSymbol],
           name: String,
           status: Deeprails::MonitorUpdateParams::Status::OrSymbol,
+          web_search: T::Boolean,
           request_options: Deeprails::RequestOptions::OrHash
         ).returns(Deeprails::MonitorUpdateResponse)
       end
       def update(
         # The ID of the monitor to edit.
         monitor_id,
-        # Description of the monitor.
+        # New description of the monitor.
         description: nil,
-        # Name of the monitor.
+        # An array of file IDs to search in the monitor's evaluations. Files must be
+        # uploaded via the DeepRails API first.
+        file_search: nil,
+        # An array of the new guardrail metrics that model input and output pairs will be
+        # evaluated on.
+        guardrail_metrics: nil,
+        # New name of the monitor.
         name: nil,
         # Status of the monitor. Can be `active` or `inactive`. Inactive monitors no
         # longer record and evaluate events.
         status: nil,
+        # Whether to enable web search for this monitor's evaluations.
+        web_search: nil,
         request_options: {}
       )
       end
@@ -112,8 +132,8 @@ module Deeprails
         # The ID of the monitor associated with this event.
         monitor_id,
         # A dictionary of inputs sent to the LLM to generate output. The dictionary must
-        # contain at least a `user_prompt` field or a `system_prompt` field. For
-        # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
+        # contain a `user_prompt` field. For ground_truth_adherence guardrail metric,
+        # `ground_truth` should be provided.
         model_input:,
         # Output generated by the LLM to be evaluated.
         model_output:,
@@ -121,8 +141,11 @@ module Deeprails
         nametag: nil,
         # Run mode for the monitor event. The run mode allows the user to optimize for
         # speed, accuracy, and cost by determining which models are used to evaluate the
-        # event. Available run modes include `precision_plus`, `precision`, `smart`, and
-        # `economy`. Defaults to `smart`.
+        # event. Available run modes (fastest to most thorough): `super_fast`, `fast`,
+        # `precision`, `precision_codex`, `precision_max`, and `precision_max_codex`.
+        # Defaults to `fast`. Note: `super_fast` does not support Web Search or File
+        # Search — if your monitor has these capabilities enabled, use a different run
+        # mode or edit the monitor to disable them.
         run_mode: nil,
         request_options: {}
       )

data/sig/deeprails/internal/stream.rbs

@@ -0,0 +1,9 @@
+module Deeprails
+  module Internal
+    class Stream[Elem]
+      include Deeprails::Internal::Type::BaseStream[Deeprails::Internal::Util::server_sent_event, Elem]
+
+      private def iterator: -> Enumerable[Elem]
+    end
+  end
+end

data/sig/deeprails/internal/type/base_stream.rbs

@@ -0,0 +1,38 @@
+module Deeprails
+  module Internal
+    module Type
+      module BaseStream[Message, Elem]
+        include Enumerable[Elem]
+
+        attr_reader status: Integer
+
+        attr_reader headers: ::Hash[String, String]
+
+        def close: -> void
+
+        private def iterator: -> Enumerable[Elem]
+
+        def each: { (Elem arg0) -> void } -> void
+
+        def to_enum: -> Enumerator[Elem]
+
+        alias enum_for to_enum
+
+        def initialize: (
+          model: Class | Deeprails::Internal::Type::Converter,
+          url: URI::Generic,
+          status: Integer,
+          headers: ::Hash[String, String],
+          response: top,
+          unwrap: Symbol
+          | Integer
+          | ::Array[Symbol | Integer]
+          | ^(top arg0) -> top,
+          stream: Enumerable[Message]
+        ) -> void
+
+        def inspect: -> String
+      end
+    end
+  end
+end

data/sig/deeprails/internal/util.rbs

@@ -106,6 +106,16 @@ module Deeprails
       JSON_CONTENT: Regexp
       JSONL_CONTENT: Regexp
 
+      def encode_query_params: (
+        ::Hash[Symbol, top] query
+      ) -> ::Hash[Symbol, top]
+
+      private def write_query_param_element!: (
+        ::Hash[Symbol, top] collection,
+        String key,
+        top element
+      ) -> nil
+
       def self?.write_multipart_content: (
         Enumerator::Yielder y,
         val: top,

data/sig/deeprails/models/defend_create_workflow_params.rbs

@@ -6,6 +6,7 @@ module Deeprails
         name: String,
         threshold_type: Deeprails::Models::DefendCreateWorkflowParams::threshold_type,
         automatic_hallucination_tolerance_levels: ::Hash[Symbol, Deeprails::Models::DefendCreateWorkflowParams::automatic_hallucination_tolerance_level],
+        context_awareness: bool,
         custom_hallucination_threshold_values: ::Hash[Symbol, Float],
         description: String,
         file_search: ::Array[String],
@@ -30,6 +31,10 @@ module Deeprails
         ::Hash[Symbol, Deeprails::Models::DefendCreateWorkflowParams::automatic_hallucination_tolerance_level]
       ) -> ::Hash[Symbol, Deeprails::Models::DefendCreateWorkflowParams::automatic_hallucination_tolerance_level]
 
+      attr_reader context_awareness: bool?
+
+      def context_awareness=: (bool) -> bool
+
       attr_reader custom_hallucination_threshold_values: ::Hash[Symbol, Float]?
 
       def custom_hallucination_threshold_values=: (
@@ -57,6 +62,7 @@ module Deeprails
         name: String,
         threshold_type: Deeprails::Models::DefendCreateWorkflowParams::threshold_type,
         ?automatic_hallucination_tolerance_levels: ::Hash[Symbol, Deeprails::Models::DefendCreateWorkflowParams::automatic_hallucination_tolerance_level],
+        ?context_awareness: bool,
         ?custom_hallucination_threshold_values: ::Hash[Symbol, Float],
         ?description: String,
         ?file_search: ::Array[String],
@@ -70,6 +76,7 @@ module Deeprails
         name: String,
         threshold_type: Deeprails::Models::DefendCreateWorkflowParams::threshold_type,
         automatic_hallucination_tolerance_levels: ::Hash[Symbol, Deeprails::Models::DefendCreateWorkflowParams::automatic_hallucination_tolerance_level],
+        context_awareness: bool,
         custom_hallucination_threshold_values: ::Hash[Symbol, Float],
         description: String,
         file_search: ::Array[String],

data/sig/deeprails/models/defend_response.rbs

@@ -115,13 +115,19 @@ module Deeprails
 
       type event =
         {
+          billing_request_id: String,
           evaluations: ::Array[Deeprails::DefendResponse::Event::Evaluation],
           event_id: String,
           improved_model_output: String,
-          improvement_tool_status: String
+          improvement_tool_status: Deeprails::Models::DefendResponse::Event::improvement_tool_status,
+          status: Deeprails::Models::DefendResponse::Event::status
         }
 
       class Event < Deeprails::Internal::Type::BaseModel
+        attr_reader billing_request_id: String?
+
+        def billing_request_id=: (String) -> String
+
         attr_reader evaluations: ::Array[Deeprails::DefendResponse::Event::Evaluation]?
 
         def evaluations=: (
@@ -136,26 +142,39 @@ module Deeprails
 
         def improved_model_output=: (String) -> String
 
-        attr_reader improvement_tool_status: String?
+        attr_reader improvement_tool_status: Deeprails::Models::DefendResponse::Event::improvement_tool_status?
+
+        def improvement_tool_status=: (
+          Deeprails::Models::DefendResponse::Event::improvement_tool_status
+        ) -> Deeprails::Models::DefendResponse::Event::improvement_tool_status
+
+        attr_reader status: Deeprails::Models::DefendResponse::Event::status?
 
-        def improvement_tool_status=: (String) -> String
+        def status=: (
+          Deeprails::Models::DefendResponse::Event::status
+        ) -> Deeprails::Models::DefendResponse::Event::status
 
         def initialize: (
+          ?billing_request_id: String,
           ?evaluations: ::Array[Deeprails::DefendResponse::Event::Evaluation],
           ?event_id: String,
           ?improved_model_output: String,
-          ?improvement_tool_status: String
+          ?improvement_tool_status: Deeprails::Models::DefendResponse::Event::improvement_tool_status,
+          ?status: Deeprails::Models::DefendResponse::Event::status
         ) -> void
 
         def to_hash: -> {
+          billing_request_id: String,
           evaluations: ::Array[Deeprails::DefendResponse::Event::Evaluation],
           event_id: String,
           improved_model_output: String,
-          improvement_tool_status: String
+          improvement_tool_status: Deeprails::Models::DefendResponse::Event::improvement_tool_status,
+          status: Deeprails::Models::DefendResponse::Event::status
         }
 
         type evaluation =
           {
+            analysis_of_failures: String,
             attempt: String,
             created_at: Time,
             error_message: String,
@@ -163,6 +182,8 @@ module Deeprails
             evaluation_status: String,
             evaluation_total_cost: Float,
             guardrail_metrics: ::Array[String],
+            improvement_tool_status: Deeprails::Models::DefendResponse::Event::Evaluation::improvement_tool_status,
+            key_improvements: ::Array[String],
             model_input: ::Hash[Symbol, top],
             model_output: String,
             modified_at: Time,
@@ -172,6 +193,10 @@ module Deeprails
           }
 
         class Evaluation < Deeprails::Internal::Type::BaseModel
+          attr_reader analysis_of_failures: String?
+
+          def analysis_of_failures=: (String) -> String
+
           attr_reader attempt: String?
 
           def attempt=: (String) -> String
@@ -200,6 +225,16 @@ module Deeprails
 
           def guardrail_metrics=: (::Array[String]) -> ::Array[String]
 
+          attr_reader improvement_tool_status: Deeprails::Models::DefendResponse::Event::Evaluation::improvement_tool_status?
+
+          def improvement_tool_status=: (
+            Deeprails::Models::DefendResponse::Event::Evaluation::improvement_tool_status
+          ) -> Deeprails::Models::DefendResponse::Event::Evaluation::improvement_tool_status
+
+          attr_reader key_improvements: ::Array[String]?
+
+          def key_improvements=: (::Array[String]) -> ::Array[String]
+
           attr_reader model_input: ::Hash[Symbol, top]?
 
           def model_input=: (::Hash[Symbol, top]) -> ::Hash[Symbol, top]
@@ -225,6 +260,7 @@ module Deeprails
           def run_mode=: (String) -> String
 
           def initialize: (
+            ?analysis_of_failures: String,
             ?attempt: String,
             ?created_at: Time,
             ?error_message: String,
@@ -232,6 +268,8 @@ module Deeprails
             ?evaluation_status: String,
             ?evaluation_total_cost: Float,
             ?guardrail_metrics: ::Array[String],
+            ?improvement_tool_status: Deeprails::Models::DefendResponse::Event::Evaluation::improvement_tool_status,
+            ?key_improvements: ::Array[String],
             ?model_input: ::Hash[Symbol, top],
             ?model_output: String,
             ?modified_at: Time,
@@ -241,6 +279,7 @@ module Deeprails
           ) -> void
 
           def to_hash: -> {
+            analysis_of_failures: String,
             attempt: String,
             created_at: Time,
             error_message: String,
@@ -248,6 +287,8 @@ module Deeprails
             evaluation_status: String,
             evaluation_total_cost: Float,
             guardrail_metrics: ::Array[String],
+            improvement_tool_status: Deeprails::Models::DefendResponse::Event::Evaluation::improvement_tool_status,
+            key_improvements: ::Array[String],
             model_input: ::Hash[Symbol, top],
             model_output: String,
             modified_at: Time,
@@ -255,10 +296,63 @@ module Deeprails
             progress: Integer,
             run_mode: String
           }
+
+          type improvement_tool_status =
+            :improved
+            | :improvement_failed
+            | :no_improvement_required
+            | :improvement_required
+
+          module ImprovementToolStatus
+            extend Deeprails::Internal::Type::Enum
+
+            IMPROVED: :improved
+            IMPROVEMENT_FAILED: :improvement_failed
+            NO_IMPROVEMENT_REQUIRED: :no_improvement_required
+            IMPROVEMENT_REQUIRED: :improvement_required
+
+            def self?.values: -> ::Array[Deeprails::Models::DefendResponse::Event::Evaluation::improvement_tool_status]
+          end
+        end
+
+        type improvement_tool_status =
+          :improved
+          | :improvement_failed
+          | :no_improvement_required
+          | :improvement_required
+
+        module ImprovementToolStatus
+          extend Deeprails::Internal::Type::Enum
+
+          IMPROVED: :improved
+          IMPROVEMENT_FAILED: :improvement_failed
+          NO_IMPROVEMENT_REQUIRED: :no_improvement_required
+          IMPROVEMENT_REQUIRED: :improvement_required
+
+          def self?.values: -> ::Array[Deeprails::Models::DefendResponse::Event::improvement_tool_status]
+        end
+
+        type status = :completed | :failed | :in_progress
+
+        module Status
+          extend Deeprails::Internal::Type::Enum
+
+          COMPLETED: :completed
+          FAILED: :failed
+          IN_PROGRESS: :in_progress
+
+          def self?.values: -> ::Array[Deeprails::Models::DefendResponse::Event::status]
         end
       end
 
-      type file = { file_id: String, file_name: String, file_size: Integer }
+      type file =
+        {
+          file_id: String,
+          file_name: String,
+          file_size: Integer,
+          presigned_url: String,
+          presigned_url_expires_at: Time
+        }
 
       class File < Deeprails::Internal::Type::BaseModel
         attr_reader file_id: String?
@@ -273,16 +367,28 @@ module Deeprails
 
         def file_size=: (Integer) -> Integer
 
+        attr_reader presigned_url: String?
+
+        def presigned_url=: (String) -> String
+
+        attr_reader presigned_url_expires_at: Time?
+
+        def presigned_url_expires_at=: (Time) -> Time
+
         def initialize: (
           ?file_id: String,
           ?file_name: String,
-          ?file_size: Integer
+          ?file_size: Integer,
+          ?presigned_url: String,
+          ?presigned_url_expires_at: Time
         ) -> void
 
         def to_hash: -> {
           file_id: String,
           file_name: String,
-          file_size: Integer
+          file_size: Integer,
+          presigned_url: String,
+          presigned_url_expires_at: Time
         }
       end
 

data/sig/deeprails/models/defend_retrieve_event_params.rbs

@@ -1,7 +1,8 @@
 module Deeprails
   module Models
     type defend_retrieve_event_params =
-      { workflow_id: String } & Deeprails::Internal::Type::request_parameters
+      { workflow_id: String, event_id: String }
+      & Deeprails::Internal::Type::request_parameters
 
     class DefendRetrieveEventParams < Deeprails::Internal::Type::BaseModel
       extend Deeprails::Internal::Type::RequestParameters::Converter
@@ -9,13 +10,17 @@ module Deeprails
 
       attr_accessor workflow_id: String
 
+      attr_accessor event_id: String
+
       def initialize: (
         workflow_id: String,
+        event_id: String,
         ?request_options: Deeprails::request_opts
       ) -> void
 
       def to_hash: -> {
         workflow_id: String,
+        event_id: String,
         request_options: Deeprails::RequestOptions
       }
     end

data/sig/deeprails/models/defend_retrieve_workflow_params.rbs

@@ -1,22 +1,27 @@
 module Deeprails
   module Models
     type defend_retrieve_workflow_params =
-      { limit: Integer } & Deeprails::Internal::Type::request_parameters
+      { workflow_id: String, limit: Integer }
+      & Deeprails::Internal::Type::request_parameters
 
     class DefendRetrieveWorkflowParams < Deeprails::Internal::Type::BaseModel
       extend Deeprails::Internal::Type::RequestParameters::Converter
       include Deeprails::Internal::Type::RequestParameters
 
+      attr_accessor workflow_id: String
+
       attr_reader limit: Integer?
 
       def limit=: (Integer) -> Integer
 
       def initialize: (
+        workflow_id: String,
         ?limit: Integer,
         ?request_options: Deeprails::request_opts
       ) -> void
 
       def to_hash: -> {
+        workflow_id: String,
         limit: Integer,
         request_options: Deeprails::RequestOptions
       }