didit-sdk 1.0.0

data/lib/didit/internal/multipart.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+
+module Didit
+  module Internal
+    # multipart/form-data bodies. Files are read into memory, so a retry can send them again.
+    # @api private
+    module Multipart
+      module_function
+
+      # @return [Array(String, String)] the body and its content type
+      def encode(fields, file_fields)
+        boundary = "didit-#{SecureRandom.hex(16)}"
+        body = +"".b
+        fields.each do |name, value|
+          next if value.nil?
+
+          if file_fields.include?(name) || value.is_a?(FilePart)
+            (value.is_a?(Array) ? value : [value]).each do |item|
+              file_part(body, boundary, name, FilePart.wrap(item, name))
+            end
+          else
+            field_part(body, boundary, name, text(value))
+          end
+        end
+        body << "--#{boundary}--\r\n"
+        [body, "multipart/form-data; boundary=#{boundary}"]
+      end
+
+      def text(value)
+        case value
+        when Hash, Array then JSON.generate(Request.json_value(value))
+        when Time then value.utc.iso8601(6)
+        when Date then value.iso8601
+        else value.to_s
+        end
+      end
+
+      def field_part(body, boundary, name, value)
+        body << "--#{boundary}\r\n"
+        body << "Content-Disposition: form-data; name=\"#{quote(name)}\"\r\n\r\n"
+        body << value.to_s.b << "\r\n"
+      end
+
+      def file_part(body, boundary, name, part)
+        body << "--#{boundary}\r\n"
+        body << "Content-Disposition: form-data; name=\"#{quote(name)}\"; filename=\"#{quote(part.filename)}\"\r\n"
+        body << "Content-Type: #{part.content_type}\r\n\r\n"
+        body << part.read << "\r\n"
+      end
+
+      # Escapes a name for a Content-Disposition parameter (WHATWG multipart rules).
+      def quote(value)
+        value.to_s.b.gsub("\r", "%0D").gsub("\n", "%0A").gsub('"', "%22")
+      end
+    end
+  end
+end

data/lib/didit/internal/request.rb

@@ -0,0 +1,322 @@
+# frozen_string_literal: true
+
+require "date"
+require "json"
+require "time"
+require "uri"
+
+module Didit
+  module Internal
+    # Builds requests and sends them with the retry policy of docs/DESIGN.md section 7.
+    # @api private
+    module Request
+      # A request with everything resolved.
+      Prepared = Struct.new(:http_method, :uri, :headers, :body, :timeout, :max_retries, :origin, :redact,
+                            keyword_init: true)
+      # A response plus how it was obtained.
+      Result = Struct.new(:response, :retries_taken, :elapsed, keyword_init: true)
+
+      IDEMPOTENT = %i[get head put delete options].freeze
+      INITIAL_RETRY_DELAY = 0.5
+      MAX_RETRY_DELAY = 8.0
+      MAX_RETRY_AFTER = 60.0
+      OPTION_KEYS = %i[timeout max_retries extra_headers extra_query extra_body].freeze
+      UNRESERVED = /[^A-Za-z0-9\-._~]/
+
+      module_function
+
+      # Per-request options (+request_options:+ of every method) merged over the client's.
+      def options(config, raw)
+        raw ||= {} # : Hash[untyped, untyped]
+        raise ConfigurationError, "request_options must be a Hash, got #{raw.class}." unless raw.is_a?(Hash)
+
+        unknown = raw.keys.map(&:to_sym) - OPTION_KEYS
+        unless unknown.empty?
+          raise ConfigurationError, "Unknown request_options #{unknown.map(&:inspect).join(", ")}; " \
+                                    "use #{OPTION_KEYS.map(&:inspect).join(", ")}."
+        end
+        opts = raw.transform_keys(&:to_sym)
+        {
+          timeout: Config.check_timeout(opts.fetch(:timeout, config.timeout)),
+          max_retries: Config.check_max_retries(opts.fetch(:max_retries, config.max_retries)),
+          extra_headers: opts[:extra_headers] || {},
+          extra_query: opts[:extra_query] || {},
+          extra_body: opts[:extra_body] || {}
+        }
+      end
+
+      # @return [Prepared]
+      def prepare(config, http_method, path, opts, path_params: {}, query: nil, body: nil, encoding: :json,
+                  file_fields: [], headers: {}, auth: :api_key, api: :verification, scope: nil,
+                  accept: "application/json")
+        params = scope == :application ? scoped(config, path_params) : path_params
+        check_credentials(config, auth)
+        base = api == :account ? config.account_base_url : config.base_url
+        uri = URI("#{base}#{encode_path(path, params)}")
+        query_string = encode_query(merge(query, opts[:extra_query]))
+        uri.query = query_string unless query_string.empty?
+        body_bytes, content_type = encode_body(body, opts[:extra_body], encoding, file_fields)
+        Prepared.new(
+          http_method: http_method, uri: uri, origin: Util.origin(base),
+          headers: build_headers(config, auth, accept: accept, content_type: content_type, op_headers: headers,
+                                               extra_headers: opts[:extra_headers]),
+          body: body_bytes, timeout: opts[:timeout], max_retries: opts[:max_retries],
+          redact: [config.api_key, config.account_token].compact
+        )
+      end
+
+      # The path parameters plus the organization and application ids of an application-scoped method.
+      def scoped(config, path_params)
+        organization_id = config.organization_id
+        application_id = config.application_id
+        unless organization_id && application_id
+          raise ConfigurationError,
+                "This method is application scoped: pass organization_id and application_id to " \
+                "Didit::Client.new (or set DIDIT_ORGANIZATION_ID and DIDIT_APPLICATION_ID)."
+        end
+
+        path_params.merge("organization_id" => organization_id, "application_id" => application_id)
+      end
+
+      def check_credentials(config, auth)
+        case auth
+        when :api_key
+          return if config.api_key
+
+          raise ConfigurationError,
+                "No API key: pass api_key: to Didit::Client.new or set the DIDIT_API_KEY environment variable."
+        when :account_token
+          return if config.account_token
+
+          raise ConfigurationError,
+                "This account method needs an account token: call didit.account.login(...) and pass its " \
+                "access_token as account_token: (didit.with_options(account_token: ...))."
+        end
+      end
+
+      # Fills +{name}+ placeholders with percent-encoded single path segments.
+      def encode_path(path, params)
+        path.gsub(/\{([^}]+)\}/) do
+          name = Regexp.last_match(1).to_s
+          value = params.fetch(name) { raise ConfigurationError, "Missing path parameter #{name}." }
+          segment = path_segment(name, value)
+          segment.b.gsub(UNRESERVED) { |c| format("%%%02X", c.ord) }.force_encoding(Encoding::US_ASCII)
+        end
+      end
+
+      def path_segment(name, value)
+        unless value.is_a?(String) || value.is_a?(Integer) || value.is_a?(Symbol)
+          raise ConfigurationError, "Path parameter #{name} must be a String, got #{value.class}."
+        end
+
+        text = value.to_s
+        raise ConfigurationError, "Path parameter #{name} must not be empty." if text.empty?
+        raise ConfigurationError, "Path parameter #{name} must not be \".\" or \"..\"." if %w[. ..].include?(text)
+
+        text.encode(Encoding::UTF_8)
+      end
+
+      def merge(values, extra)
+        # @type var out: Hash[String, untyped]
+        out = {}
+        (values || {}).each { |k, v| out[k.to_s] = v unless v.equal?(OMIT) }
+        (extra || {}).each { |k, v| out[k.to_s] = v }
+        out
+      end
+
+      # Query values: nil and omitted values are left out, arrays repeat the key.
+      def encode_query(values)
+        # @type var pairs: Array[[String, String]]
+        pairs = []
+        values.each do |key, value|
+          Array(value.is_a?(Hash) ? JSON.generate(json_value(value)) : value).each do |item|
+            pairs << [key, query_value(item)] unless item.nil? || item.equal?(OMIT)
+          end
+        end
+        URI.encode_www_form(pairs)
+      end
+
+      def query_value(value)
+        case value
+        when Time then value.utc.iso8601(6)
+        when Date then value.iso8601
+        else value.to_s
+        end
+      end
+
+      # @return [Array(String, String), Array(nil, nil)] body bytes and content type
+      def encode_body(body, extra_body, encoding, file_fields)
+        return [nil, nil] if body.nil? && (extra_body.nil? || extra_body.empty?)
+
+        fields = merge(body, extra_body)
+        multipart = encoding == :multipart ||
+                    (encoding == :auto && file_fields.any? { |f| !fields[f].nil? })
+        if multipart
+          Multipart.encode(fields, file_fields)
+        else
+          [JSON.generate(json_value(fields)), "application/json"]
+        end
+      end
+
+      # A value ready for JSON: string keys, Time and Date as ISO 8601.
+      def json_value(value)
+        case value
+        when Hash
+          # @type var out: Hash[String, untyped]
+          out = {}
+          value.each { |k, v| out[k.to_s] = json_value(v) unless v.equal?(OMIT) }
+          out
+        when Array then value.map { |v| json_value(v) }
+        when Time then value.utc.iso8601(6)
+        when Date then value.iso8601
+        when Symbol then value.to_s
+        when Didit::Model then json_value(value.to_h)
+        when FilePart
+          raise ConfigurationError, "A file can only be sent to an operation that accepts uploads."
+        else value
+        end
+      end
+
+      def build_headers(config, auth, accept:, content_type:, op_headers:, extra_headers:)
+        # @type var pairs: Array[[untyped, untyped]]
+        pairs = [["Accept", accept], ["User-Agent", Util.user_agent], ["X-Didit-Client", Util.client_header]]
+        pairs.concat(config.default_headers.to_a)
+        credential = credential_header(config, auth)
+        pairs << credential if credential
+        pairs << ["Content-Type", content_type] if content_type
+        pairs.concat(op_headers.reject { |_, value| value.equal?(OMIT) }.to_a)
+        pairs.concat(extra_headers.to_a)
+        merge_headers(pairs)
+      end
+
+      # Header names are case-insensitive: a later value replaces an earlier one whatever its
+      # case, so no header is sent twice; nil removes the header.
+      def merge_headers(pairs)
+        # @type var headers: Hash[String, String]
+        headers = {}
+        pairs.each do |name, value|
+          name = name.to_s
+          headers.delete_if { |key, _| key.casecmp?(name) }
+          headers[name] = value.to_s unless value.nil?
+        end
+        headers
+      end
+
+      # The credential header for +auth+, sent only to the origin the request was built for.
+      def credential_header(config, auth)
+        case auth
+        when :api_key then ["x-api-key", config.api_key.to_s]
+        when :account_token then ["Authorization", "Bearer #{config.account_token}"]
+        end
+      end
+
+      # One try: the response, or the failure and whether any body chunk was already streamed.
+      Attempt = Struct.new(:response, :failure, :streamed, :elapsed, keyword_init: true)
+
+      # Sends +prepared+, retrying as docs/DESIGN.md section 7 says.
+      #
+      # @return [Result]
+      def perform(config, transport, prepared, &block)
+        retries = 0
+        loop do
+          attempt = attempt(transport, prepared, &block)
+          delay = retry_delay(config, prepared, attempt, retries)
+          if delay.nil?
+            failure = attempt.failure
+            raise connection_error(prepared, failure), cause: failure if failure
+
+            response = attempt.response or raise Transport::Failure.new(IOError.new("no response"), sent: true)
+            return Result.new(response: response, retries_taken: retries, elapsed: attempt.elapsed)
+          end
+          retries += 1
+          Util.pause(delay)
+        end
+      end
+
+      def attempt(transport, prepared)
+        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        result = Attempt.new(streamed: false)
+        # @type var stream: (^(String) -> void)?
+        stream = nil
+        if block_given?
+          stream = lambda do |chunk|
+            result.streamed = true
+            yield(chunk)
+          end
+        end
+        begin
+          result.response = transport.execute(prepared.http_method, prepared.uri, prepared.headers, prepared.body,
+                                              prepared.timeout, &stream)
+        rescue Transport::Failure => e
+          result.failure = e
+        end
+        result.elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
+        result
+      end
+
+      # Seconds to wait before the next try, or nil when this attempt is final.
+      def retry_delay(config, prepared, attempt, retries)
+        left = prepared.max_retries - retries
+        if (failure = attempt.failure)
+          return nil if attempt.streamed || left <= 0 || !retry_failure?(prepared.http_method, failure)
+
+          delay = backoff(retries)
+          Log.retry(config, prepared, failure.timeout? ? "timed out" : failure.error.class.name, delay, left)
+          return delay
+        end
+        response = attempt.response
+        return nil unless response
+
+        Log.response(config, prepared, response, attempt.elapsed)
+        return nil if left <= 0 || !retry_status?(prepared.http_method, response.status)
+
+        delay = retry_after(response.headers["retry-after"]) || backoff(retries)
+        Log.retry(config, prepared, "HTTP #{response.status}", delay, left)
+        delay
+      end
+
+      def retry_status?(http_method, status)
+        return true if status == 429
+        return false unless IDEMPOTENT.include?(http_method)
+
+        status == 408 || status.between?(500, 599)
+      end
+
+      def retry_failure?(http_method, failure)
+        !failure.sent || IDEMPOTENT.include?(http_method)
+      end
+
+      # Exponential backoff from 0.5 s with full jitter, capped at 8 s.
+      def backoff(retries_taken)
+        Kernel.rand * [MAX_RETRY_DELAY, INITIAL_RETRY_DELAY * (2**retries_taken)].min
+      end
+
+      # Seconds to wait from a Retry-After header (seconds or HTTP date), capped at 60.
+      def retry_after(value, now: Time.now)
+        return nil if value.nil?
+
+        text = value.strip
+        seconds = text.match?(/\A\d+(\.\d+)?\z/) ? text.to_f : http_date_seconds(text, now)
+        seconds&.clamp(0.0, MAX_RETRY_AFTER)
+      end
+
+      def http_date_seconds(text, now)
+        Time.httpdate(text) - now
+      rescue ArgumentError
+        nil
+      end
+
+      def connection_error(prepared, failure)
+        url = Util.display_url(prepared.uri)
+        method = prepared.http_method.to_s.upcase
+        if failure.timeout?
+          APITimeoutError.new("Request timed out (#{method} #{url}).", http_method: method, url: url)
+        else
+          detail = Util.redact(failure.error.message.to_s, prepared.redact)
+          APIConnectionError.new("Connection error (#{method} #{url}): #{failure.error.class}: #{detail}",
+                                 http_method: method, url: url)
+        end
+      end
+    end
+  end
+end

data/lib/didit/internal/transport.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "openssl"
+require "uri"
+require "zlib"
+
+module Didit
+  module Internal
+    # Sends HTTP requests over persistent net/http connections, one small pool per origin.
+    # Thread safe and fork safe: a child process never reuses its parent's sockets.
+    # @api private
+    class Transport
+      # What came back: status, headers (lower-case names) and the body bytes.
+      Response = Struct.new(:status, :headers, :body, keyword_init: true)
+
+      # Raised by {#execute} when no response arrived; +sent+ says whether the request may
+      # have reached the server.
+      class Failure < StandardError
+        attr_reader :error, :sent
+
+        def initialize(error, sent:)
+          super(error.message)
+          @error = error
+          @sent = sent
+        end
+
+        def timeout?
+          TIMEOUT_ERRORS.any? { |klass| error.is_a?(klass) }
+        end
+      end
+
+      # Errors that happen before any byte of the request leaves this process.
+      NOT_SENT_ERRORS = [
+        Net::OpenTimeout, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::EADDRNOTAVAIL,
+        SocketError
+      ].freeze
+      # Errors after which the request may have reached the server.
+      TRANSPORT_ERRORS = [
+        Net::ReadTimeout, Net::WriteTimeout, Timeout::Error, EOFError, IOError, SystemCallError,
+        OpenSSL::SSL::SSLError, Net::HTTPBadResponse, Net::ProtocolError, Zlib::Error
+      ].freeze
+      TIMEOUT_ERRORS = [Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout, Timeout::Error, Errno::ETIMEDOUT].freeze
+      MAX_IDLE = 8
+
+      METHODS = {
+        get: Net::HTTP::Get, post: Net::HTTP::Post, put: Net::HTTP::Put, patch: Net::HTTP::Patch,
+        delete: Net::HTTP::Delete, head: Net::HTTP::Head, options: Net::HTTP::Options
+      }.freeze
+
+      # @param http_client [#call, nil] returns a configured, not yet started Net::HTTP for a URI
+      def initialize(http_client: nil)
+        @http_client = http_client
+        @mutex = Mutex.new
+        @idle = {}
+        @pid = Process.pid
+      end
+
+      attr_reader :http_client
+
+      # Sends one request. With a block, a 2xx body is yielded in chunks instead of returned.
+      #
+      # @return [Response]
+      # @raise [Failure]
+      def execute(http_method, uri, headers, body, timeout, &)
+        http = checkout(uri, timeout)
+        reusable = false
+        begin
+          request = build(http_method, uri, headers, body)
+          response = perform(http, request, &)
+          reusable = response.headers["connection"]&.casecmp?("close") != true
+          response
+        rescue *NOT_SENT_ERRORS => e
+          raise Failure.new(e, sent: false)
+        rescue *TRANSPORT_ERRORS => e
+          raise Failure.new(e, sent: true)
+        ensure
+          reusable ? checkin(uri, http) : discard(http)
+        end
+      end
+
+      # Closes every idle connection.
+      def close
+        sessions = @mutex.synchronize do
+          all = @idle.values.flatten
+          @idle = {}
+          all
+        end
+        sessions.each { |http| discard(http) }
+        nil
+      end
+
+      private
+
+      def perform(http, request, &block)
+        # @type var result: Response?
+        result = nil
+        http.request(request) do |res|
+          # @type var headers: Hash[String, String]
+          headers = {}
+          res.each_header { |name, value| headers[name.downcase] = value }
+          status = res.code.to_i
+          if block && status.between?(200, 299)
+            res.read_body(&block)
+            result = Response.new(status: status, headers: headers, body: nil)
+          else
+            result = Response.new(status: status, headers: headers, body: res.read_body.to_s.b)
+          end
+        end
+        result || raise(Net::HTTPBadResponse, "no response")
+      end
+
+      def build(http_method, uri, headers, body)
+        klass = METHODS.fetch(http_method)
+        request = klass.new(target(uri), headers)
+        request.body = body if body
+        request
+      end
+
+      # The request target: path ("/" when empty) and query.
+      def target(uri)
+        path = uri.path.to_s.empty? ? "/" : uri.path.to_s
+        uri.query ? "#{path}?#{uri.query}" : path
+      end
+
+      def checkout(uri, timeout)
+        key = pool_key(uri)
+        http = @mutex.synchronize do
+          reset_after_fork
+          @idle[key]&.pop
+        end
+        http ||= new_session(uri)
+        configure(http, timeout)
+        start(http) unless http.started?
+        http
+      end
+
+      def start(http)
+        http.start
+      rescue *NOT_SENT_ERRORS, *TRANSPORT_ERRORS => e
+        # Nothing was sent: the connection itself failed.
+        discard(http)
+        raise Failure.new(e, sent: false)
+      end
+
+      def checkin(uri, http)
+        keep = @mutex.synchronize do
+          list = (@idle[pool_key(uri)] ||= [])
+          if @pid == Process.pid && list.size < MAX_IDLE
+            list.push(http)
+            true
+          else
+            false
+          end
+        end
+        discard(http) unless keep
+      end
+
+      def discard(http)
+        http.finish if http.started?
+      rescue IOError, SystemCallError, OpenSSL::SSL::SSLError
+        nil
+      end
+
+      def reset_after_fork
+        return if @pid == Process.pid
+
+        # The sockets belong to the parent process: forget them without closing them.
+        @idle = {}
+        @pid = Process.pid
+      end
+
+      def new_session(uri)
+        factory = @http_client
+        http = factory ? factory.call(uri) : Net::HTTP.new(uri.hostname.to_s, uri.port)
+        unless http.is_a?(Net::HTTP)
+          raise ConfigurationError, "http_client must return a Net::HTTP instance, got #{http.class}."
+        end
+
+        http.use_ssl = true if uri.scheme == "https" && !http.use_ssl?
+        http
+      end
+
+      def configure(http, timeout)
+        http.open_timeout = timeout
+        http.read_timeout = timeout
+        http.write_timeout = timeout
+        http.ssl_timeout = timeout if http.use_ssl?
+        # The SDK decides about retries (docs/DESIGN.md section 7), net/http never does.
+        http.max_retries = 0
+      end
+
+      def pool_key(uri)
+        "#{uri.scheme}://#{uri.host}:#{uri.port}"
+      end
+    end
+  end
+end

data/lib/didit/internal/util.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Didit
+  module Internal
+    # Small helpers shared by the client, webhooks and Authentication.
+    # @api private
+    module Util
+      REQUEST_ID_HEADERS = %w[x-request-id request-id x-amzn-requestid x-correlation-id].freeze
+      REDACTED = "[redacted]"
+
+      module_function
+
+      # +User-Agent: didit-ruby/<version> (<engine> <version>; <platform>)+ (docs/DESIGN.md section 6).
+      def user_agent
+        "didit-ruby/#{Didit::VERSION} (#{RUBY_ENGINE} #{RUBY_ENGINE_VERSION}; #{RUBY_PLATFORM})"
+      end
+
+      # +X-Didit-Client: ruby/<version>+.
+      def client_header
+        "ruby/#{Didit::VERSION}"
+      end
+
+      # The request id from response headers (names in lower case), if any.
+      def request_id(headers)
+        REQUEST_ID_HEADERS.each do |name|
+          value = headers[name]
+          return value unless value.nil? || value.empty?
+        end
+        nil
+      end
+
+      # Replaces every secret in +value+ (a String, or Hash/Array of them) with "[redacted]".
+      def redact(value, secrets)
+        secrets = secrets.compact.reject(&:empty?)
+        return value if secrets.empty?
+
+        redact_value(value, secrets)
+      end
+
+      def redact_value(value, secrets)
+        case value
+        when String then secrets.reduce(value) { |text, secret| text.gsub(secret, REDACTED) }
+        when Hash then value.to_h { |k, v| [redact_value(k, secrets), redact_value(v, secrets)] }
+        when Array then value.map { |v| redact_value(v, secrets) }
+        else value
+        end
+      end
+
+      # +scheme://host[:port]+ with the default port left out.
+      def origin(uri)
+        uri = URI(uri) unless uri.is_a?(URI::Generic)
+        default = { "http" => 80, "https" => 443 }[uri.scheme]
+        port = uri.port.nil? || uri.port == default ? "" : ":#{uri.port}"
+        "#{uri.scheme}://#{uri.host&.downcase}#{port}"
+      end
+
+      # The URL without query string, fragment or credentials (for errors and logs).
+      def display_url(uri)
+        uri = URI(uri) unless uri.is_a?(URI::Generic)
+        "#{origin(uri)}#{uri.path}"
+      end
+
+      # Sleeps between retries (a separate method, so tests can skip the wait).
+      def pause(seconds)
+        Kernel.sleep(seconds) if seconds.positive?
+      end
+
+      # Whether the environment variable asks for debug logging.
+      def debug_from_env?
+        ENV.fetch("DIDIT_LOG", "").strip.casecmp?("debug")
+      end
+    end
+  end
+end

data/lib/didit/internal.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Didit
+  # Implementation details. Nothing under this module is public API.
+  # @api private
+  module Internal
+    autoload :Config, "didit/internal/config"
+    autoload :Errors, "didit/internal/errors"
+    autoload :Log, "didit/internal/log"
+    autoload :Multipart, "didit/internal/multipart"
+    autoload :Request, "didit/internal/request"
+    autoload :Transport, "didit/internal/transport"
+    autoload :Util, "didit/internal/util"
+  end
+end