deeprails 0.1.0

data/lib/deeprails/resources/evaluate.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Deeprails
+  module Resources
+    class Evaluate
+      # Some parameter documentations has been truncated, see
+      # {Deeprails::Models::EvaluateCreateParams} for more details.
+      #
+      # Use this endpoint to evaluate a model's input and output pair against selected
+      # guardrail metrics
+      #
+      # @overload create(model_input:, model_output:, run_mode:, guardrail_metrics: nil, model_used: nil, nametag: nil, request_options: {})
+      #
+      # @param model_input [Deeprails::Models::EvaluateCreateParams::ModelInput] A dictionary of inputs sent to the LLM to generate output. This must contain a
+      #
+      # @param model_output [String] Output generated by the LLM to be evaluated.
+      #
+      # @param run_mode [Symbol, Deeprails::Models::EvaluateCreateParams::RunMode] Run mode for the evaluation. The run mode allows the user to optimize for speed
+      #
+      # @param guardrail_metrics [Array<Symbol, Deeprails::Models::EvaluateCreateParams::GuardrailMetric>] An array of guardrail metrics that the model input and output pair will be evalu
+      #
+      # @param model_used [String] Model ID used to generate the output, like `gpt-4o` or `o3`.
+      #
+      # @param nametag [String] An optional, user-defined tag for the evaluation.
+      #
+      # @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}, nil]
+      #
+      # @return [Deeprails::Models::Evaluation]
+      #
+      # @see Deeprails::Models::EvaluateCreateParams
+      def create(params)
+        parsed, options = Deeprails::EvaluateCreateParams.dump_request(params)
+        @client.request(
+          method: :post,
+          path: "evaluate",
+          body: parsed,
+          model: Deeprails::Evaluation,
+          options: options
+        )
+      end
+
+      # Retrieve the evaluation record for a given evaluation ID.
+      #
+      # @overload retrieve(eval_id, request_options: {})
+      #
+      # @param eval_id [String] The ID of the evaluation to retrieve.
+      #
+      # @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}, nil]
+      #
+      # @return [Deeprails::Models::Evaluation]
+      #
+      # @see Deeprails::Models::EvaluateRetrieveParams
+      def retrieve(eval_id, params = {})
+        @client.request(
+          method: :get,
+          path: ["evaluate/%1$s", eval_id],
+          model: Deeprails::Evaluation,
+          options: params[:request_options]
+        )
+      end
+
+      # @api private
+      #
+      # @param client [Deeprails::Client]
+      def initialize(client:)
+        @client = client
+      end
+    end
+  end
+end

data/lib/deeprails/resources/monitor.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module Deeprails
+  module Resources
+    class Monitor
+      # Create a new monitor to evaluate model inputs and outputs using guardrails.
+      #
+      # @overload create(name:, description: nil, request_options: {})
+      #
+      # @param name [String] Name of the new monitor.
+      #
+      # @param description [String] Description of the new monitor.
+      #
+      # @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}, nil]
+      #
+      # @return [Deeprails::Models::APIResponse]
+      #
+      # @see Deeprails::Models::MonitorCreateParams
+      def create(params)
+        parsed, options = Deeprails::MonitorCreateParams.dump_request(params)
+        @client.request(
+          method: :post,
+          path: "monitor",
+          body: parsed,
+          model: Deeprails::APIResponse,
+          options: options
+        )
+      end
+
+      # Retrieve the details and evaluations associated with a specific monitor.
+      #
+      # @overload retrieve(monitor_id, limit: nil, request_options: {})
+      #
+      # @param monitor_id [String] The ID of the monitor to retrieve.
+      #
+      # @param limit [Integer] Limit the returned events associated with this monitor. Defaults to 10.
+      #
+      # @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}, nil]
+      #
+      # @return [Deeprails::Models::MonitorRetrieveResponse]
+      #
+      # @see Deeprails::Models::MonitorRetrieveParams
+      def retrieve(monitor_id, params = {})
+        parsed, options = Deeprails::MonitorRetrieveParams.dump_request(params)
+        @client.request(
+          method: :get,
+          path: ["monitor/%1$s", monitor_id],
+          query: parsed,
+          model: Deeprails::Models::MonitorRetrieveResponse,
+          options: options
+        )
+      end
+
+      # Some parameter documentations has been truncated, see
+      # {Deeprails::Models::MonitorUpdateParams} for more details.
+      #
+      # Update the name, description, or status of an existing monitor.
+      #
+      # @overload update(monitor_id, description: nil, monitor_status: nil, name: nil, request_options: {})
+      #
+      # @param monitor_id [String] The ID of the monitor to edit.
+      #
+      # @param description [String] Description of the monitor.
+      #
+      # @param monitor_status [Symbol, Deeprails::Models::MonitorUpdateParams::MonitorStatus] Status of the monitor. Can be `active` or `inactive`. Inactive monitors no lon
+      #
+      # @param name [String] Name of the monitor.
+      #
+      # @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}, nil]
+      #
+      # @return [Deeprails::Models::APIResponse]
+      #
+      # @see Deeprails::Models::MonitorUpdateParams
+      def update(monitor_id, params = {})
+        parsed, options = Deeprails::MonitorUpdateParams.dump_request(params)
+        @client.request(
+          method: :put,
+          path: ["monitor/%1$s", monitor_id],
+          body: parsed,
+          model: Deeprails::APIResponse,
+          options: options
+        )
+      end
+
+      # Some parameter documentations has been truncated, see
+      # {Deeprails::Models::MonitorSubmitEventParams} for more details.
+      #
+      # Submit a model input and output pair to a monitor for evaluation.
+      #
+      # @overload submit_event(monitor_id, guardrail_metrics:, model_input:, model_output:, model_used: nil, nametag: nil, run_mode: nil, request_options: {})
+      #
+      # @param monitor_id [String] The ID of the monitor associated with this event.
+      #
+      # @param guardrail_metrics [Array<Symbol, Deeprails::Models::MonitorSubmitEventParams::GuardrailMetric>] An array of guardrail metrics that the model input and output pair will be evalu
+      #
+      # @param model_input [Deeprails::Models::MonitorSubmitEventParams::ModelInput] A dictionary of inputs sent to the LLM to generate output. This must contain a
+      #
+      # @param model_output [String] Output generated by the LLM to be evaluated.
+      #
+      # @param model_used [String] Model ID used to generate the output, like `gpt-4o` or `o3`.
+      #
+      # @param nametag [String] An optional, user-defined tag for the event.
+      #
+      # @param run_mode [Symbol, Deeprails::Models::MonitorSubmitEventParams::RunMode] Run mode for the monitor event. The run mode allows the user to optimize for sp
+      #
+      # @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}, nil]
+      #
+      # @return [Deeprails::Models::MonitorSubmitEventResponse]
+      #
+      # @see Deeprails::Models::MonitorSubmitEventParams
+      def submit_event(monitor_id, params)
+        parsed, options = Deeprails::MonitorSubmitEventParams.dump_request(params)
+        @client.request(
+          method: :post,
+          path: ["monitor/%1$s/events", monitor_id],
+          body: parsed,
+          model: Deeprails::Models::MonitorSubmitEventResponse,
+          options: options
+        )
+      end
+
+      # @api private
+      #
+      # @param client [Deeprails::Client]
+      def initialize(client:)
+        @client = client
+      end
+    end
+  end
+end

data/lib/deeprails/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Deeprails
+  VERSION = "0.1.0"
+end

data/lib/deeprails.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+# Standard libraries.
+# rubocop:disable Lint/RedundantRequireStatement
+require "English"
+require "cgi"
+require "date"
+require "erb"
+require "etc"
+require "json"
+require "net/http"
+require "pathname"
+require "rbconfig"
+require "securerandom"
+require "set"
+require "stringio"
+require "time"
+require "uri"
+# rubocop:enable Lint/RedundantRequireStatement
+
+# We already ship the preferred sorbet manifests in the package itself.
+# `tapioca` currently does not offer us a way to opt out of unnecessary compilation.
+if Object.const_defined?(:Tapioca) &&
+   caller.chain([$PROGRAM_NAME]).chain(ARGV).any?(/tapioca/) &&
+   ARGV.none?(/dsl/)
+  return
+end
+
+# Gems.
+require "connection_pool"
+
+# Package files.
+require_relative "deeprails/version"
+require_relative "deeprails/internal/util"
+require_relative "deeprails/internal/type/converter"
+require_relative "deeprails/internal/type/unknown"
+require_relative "deeprails/internal/type/boolean"
+require_relative "deeprails/internal/type/file_input"
+require_relative "deeprails/internal/type/enum"
+require_relative "deeprails/internal/type/union"
+require_relative "deeprails/internal/type/array_of"
+require_relative "deeprails/internal/type/hash_of"
+require_relative "deeprails/internal/type/base_model"
+require_relative "deeprails/internal/type/base_page"
+require_relative "deeprails/internal/type/request_parameters"
+require_relative "deeprails/internal"
+require_relative "deeprails/request_options"
+require_relative "deeprails/file_part"
+require_relative "deeprails/errors"
+require_relative "deeprails/internal/transport/base_client"
+require_relative "deeprails/internal/transport/pooled_net_requester"
+require_relative "deeprails/client"
+require_relative "deeprails/models/api_response"
+require_relative "deeprails/models/defend_create_workflow_params"
+require_relative "deeprails/models/defend_response"
+require_relative "deeprails/models/defend_retrieve_event_params"
+require_relative "deeprails/models/defend_retrieve_workflow_params"
+require_relative "deeprails/models/defend_submit_event_params"
+require_relative "deeprails/models/defend_update_workflow_params"
+require_relative "deeprails/models/evaluate_create_params"
+require_relative "deeprails/models/evaluate_retrieve_params"
+require_relative "deeprails/models/evaluation"
+require_relative "deeprails/models/monitor_create_params"
+require_relative "deeprails/models/monitor_retrieve_params"
+require_relative "deeprails/models/monitor_retrieve_response"
+require_relative "deeprails/models/monitor_submit_event_params"
+require_relative "deeprails/models/monitor_submit_event_response"
+require_relative "deeprails/models/monitor_update_params"
+require_relative "deeprails/models/workflow_event_response"
+require_relative "deeprails/models"
+require_relative "deeprails/resources/defend"
+require_relative "deeprails/resources/evaluate"
+require_relative "deeprails/resources/monitor"

data/manifest.yaml

@@ -0,0 +1,15 @@
+dependencies:
+  - English
+  - cgi
+  - date
+  - erb
+  - etc
+  - json
+  - net/http
+  - pathname
+  - rbconfig
+  - securerandom
+  - set
+  - stringio
+  - time
+  - uri

data/rbi/deeprails/client.rbi

@@ -0,0 +1,55 @@
+# typed: strong
+
+module Deeprails
+  class Client < Deeprails::Internal::Transport::BaseClient
+    DEFAULT_MAX_RETRIES = 2
+
+    DEFAULT_TIMEOUT_IN_SECONDS = T.let(60.0, Float)
+
+    DEFAULT_INITIAL_RETRY_DELAY = T.let(0.5, Float)
+
+    DEFAULT_MAX_RETRY_DELAY = T.let(8.0, Float)
+
+    sig { returns(String) }
+    attr_reader :api_key
+
+    sig { returns(Deeprails::Resources::Defend) }
+    attr_reader :defend
+
+    sig { returns(Deeprails::Resources::Monitor) }
+    attr_reader :monitor
+
+    sig { returns(Deeprails::Resources::Evaluate) }
+    attr_reader :evaluate
+
+    # @api private
+    sig { override.returns(T::Hash[String, String]) }
+    private def auth_headers
+    end
+
+    # Creates and returns a new client for interacting with the API.
+    sig do
+      params(
+        api_key: T.nilable(String),
+        base_url: T.nilable(String),
+        max_retries: Integer,
+        timeout: Float,
+        initial_retry_delay: Float,
+        max_retry_delay: Float
+      ).returns(T.attached_class)
+    end
+    def self.new(
+      # Defaults to `ENV["DEEPRAILS_API_KEY"]`
+      api_key: ENV["DEEPRAILS_API_KEY"],
+      # Override the default base URL for the API, e.g.,
+      # `"https://api.example.com/v2/"`. Defaults to `ENV["DEEPRAILS_BASE_URL"]`
+      base_url: ENV["DEEPRAILS_BASE_URL"],
+      # Max number of retries to attempt after a failed retryable request.
+      max_retries: Deeprails::Client::DEFAULT_MAX_RETRIES,
+      timeout: Deeprails::Client::DEFAULT_TIMEOUT_IN_SECONDS,
+      initial_retry_delay: Deeprails::Client::DEFAULT_INITIAL_RETRY_DELAY,
+      max_retry_delay: Deeprails::Client::DEFAULT_MAX_RETRY_DELAY
+    )
+    end
+  end
+end

data/rbi/deeprails/errors.rbi

@@ -0,0 +1,205 @@
+# typed: strong
+
+module Deeprails
+  module Errors
+    class Error < StandardError
+      sig { returns(T.nilable(StandardError)) }
+      attr_accessor :cause
+    end
+
+    class ConversionError < Deeprails::Errors::Error
+      sig { returns(T.nilable(StandardError)) }
+      def cause
+      end
+
+      # @api private
+      sig do
+        params(
+          on: T::Class[StandardError],
+          method: Symbol,
+          target: T.anything,
+          value: T.anything,
+          cause: T.nilable(StandardError)
+        ).returns(T.attached_class)
+      end
+      def self.new(on:, method:, target:, value:, cause: nil)
+      end
+    end
+
+    class APIError < Deeprails::Errors::Error
+      sig { returns(URI::Generic) }
+      attr_accessor :url
+
+      sig { returns(T.nilable(Integer)) }
+      attr_accessor :status
+
+      sig { returns(T.nilable(T::Hash[String, String])) }
+      attr_accessor :headers
+
+      sig { returns(T.nilable(T.anything)) }
+      attr_accessor :body
+
+      # @api private
+      sig do
+        params(
+          url: URI::Generic,
+          status: T.nilable(Integer),
+          headers: T.nilable(T::Hash[String, String]),
+          body: T.nilable(Object),
+          request: NilClass,
+          response: NilClass,
+          message: T.nilable(String)
+        ).returns(T.attached_class)
+      end
+      def self.new(
+        url:,
+        status: nil,
+        headers: nil,
+        body: nil,
+        request: nil,
+        response: nil,
+        message: nil
+      )
+      end
+    end
+
+    class APIConnectionError < Deeprails::Errors::APIError
+      sig { returns(NilClass) }
+      attr_accessor :status
+
+      sig { returns(NilClass) }
+      attr_accessor :body
+
+      # @api private
+      sig do
+        params(
+          url: URI::Generic,
+          status: NilClass,
+          headers: T.nilable(T::Hash[String, String]),
+          body: NilClass,
+          request: NilClass,
+          response: NilClass,
+          message: T.nilable(String)
+        ).returns(T.attached_class)
+      end
+      def self.new(
+        url:,
+        status: nil,
+        headers: nil,
+        body: nil,
+        request: nil,
+        response: nil,
+        message: "Connection error."
+      )
+      end
+    end
+
+    class APITimeoutError < Deeprails::Errors::APIConnectionError
+      # @api private
+      sig do
+        params(
+          url: URI::Generic,
+          status: NilClass,
+          headers: T.nilable(T::Hash[String, String]),
+          body: NilClass,
+          request: NilClass,
+          response: NilClass,
+          message: T.nilable(String)
+        ).returns(T.attached_class)
+      end
+      def self.new(
+        url:,
+        status: nil,
+        headers: nil,
+        body: nil,
+        request: nil,
+        response: nil,
+        message: "Request timed out."
+      )
+      end
+    end
+
+    class APIStatusError < Deeprails::Errors::APIError
+      # @api private
+      sig do
+        params(
+          url: URI::Generic,
+          status: Integer,
+          headers: T.nilable(T::Hash[String, String]),
+          body: T.nilable(Object),
+          request: NilClass,
+          response: NilClass,
+          message: T.nilable(String)
+        ).returns(T.attached_class)
+      end
+      def self.for(
+        url:,
+        status:,
+        headers:,
+        body:,
+        request:,
+        response:,
+        message: nil
+      )
+      end
+
+      sig { returns(Integer) }
+      attr_accessor :status
+
+      # @api private
+      sig do
+        params(
+          url: URI::Generic,
+          status: Integer,
+          headers: T.nilable(T::Hash[String, String]),
+          body: T.nilable(Object),
+          request: NilClass,
+          response: NilClass,
+          message: T.nilable(String)
+        ).returns(T.attached_class)
+      end
+      def self.new(
+        url:,
+        status:,
+        headers:,
+        body:,
+        request:,
+        response:,
+        message: nil
+      )
+      end
+    end
+
+    class BadRequestError < Deeprails::Errors::APIStatusError
+      HTTP_STATUS = 400
+    end
+
+    class AuthenticationError < Deeprails::Errors::APIStatusError
+      HTTP_STATUS = 401
+    end
+
+    class PermissionDeniedError < Deeprails::Errors::APIStatusError
+      HTTP_STATUS = 403
+    end
+
+    class NotFoundError < Deeprails::Errors::APIStatusError
+      HTTP_STATUS = 404
+    end
+
+    class ConflictError < Deeprails::Errors::APIStatusError
+      HTTP_STATUS = 409
+    end
+
+    class UnprocessableEntityError < Deeprails::Errors::APIStatusError
+      HTTP_STATUS = 422
+    end
+
+    class RateLimitError < Deeprails::Errors::APIStatusError
+      HTTP_STATUS = 429
+    end
+
+    class InternalServerError < Deeprails::Errors::APIStatusError
+      HTTP_STATUS = T.let((500..), T::Range[Integer])
+    end
+  end
+end

data/rbi/deeprails/file_part.rbi

@@ -0,0 +1,37 @@
+# typed: strong
+
+module Deeprails
+  class FilePart
+    sig { returns(T.any(Pathname, StringIO, IO, String)) }
+    attr_reader :content
+
+    sig { returns(T.nilable(String)) }
+    attr_reader :content_type
+
+    sig { returns(T.nilable(String)) }
+    attr_reader :filename
+
+    # @api private
+    sig { returns(String) }
+    private def read
+    end
+
+    sig { params(a: T.anything).returns(String) }
+    def to_json(*a)
+    end
+
+    sig { params(a: T.anything).returns(String) }
+    def to_yaml(*a)
+    end
+
+    sig do
+      params(
+        content: T.any(Pathname, StringIO, IO, String),
+        filename: T.nilable(T.any(Pathname, String)),
+        content_type: T.nilable(String)
+      ).returns(T.attached_class)
+    end
+    def self.new(content, filename: nil, content_type: nil)
+    end
+  end
+end