typhoeus 0.4.2 → 1.4.0

data/lib/typhoeus/easy_factory.rb

@@ -0,0 +1,180 @@
+require 'set'
+
+module Typhoeus
+
+  # This is a Factory for easies to be used in the hydra.
+  # Before an easy is ready to be added to a multi the
+  # on_complete callback to be set.
+  # This is done by this class.
+  #
+  # @api private
+  class EasyFactory
+
+    RENAMED_OPTIONS =  {
+          :auth_method => :httpauth,
+          :connect_timeout => :connecttimeout,
+          :encoding => :accept_encoding,
+          :follow_location => :followlocation,
+          :max_redirects => :maxredirs,
+          :proxy_type => :proxytype,
+          :ssl_cacert => :cainfo,
+          :ssl_capath => :capath,
+          :ssl_cert => :sslcert,
+          :ssl_cert_type => :sslcerttype,
+          :ssl_key => :sslkey,
+          :ssl_key_password => :keypasswd,
+          :ssl_key_type => :sslkeytype,
+          :ssl_version => :sslversion,
+      }
+
+    CHANGED_OPTIONS =  {
+          :disable_ssl_host_verification => :ssl_verifyhost,
+          :disable_ssl_peer_verification => :ssl_verifypeer,
+          :proxy_auth_method => :proxyauth,
+      }
+
+    REMOVED_OPTIONS =  Set.new([:cache_key_basis, :cache_timeout, :user_agent])
+
+    SANITIZE_IGNORE  = Set.new([:method, :cache_ttl, :cache])
+    SANITIZE_TIMEOUT = Set.new([:timeout_ms, :connecttimeout_ms])
+
+    # Returns the request provided.
+    #
+    # @return [ Typhoeus::Request ]
+    attr_reader :request
+
+    # Returns the hydra provided.
+    #
+    # @return [ Typhoeus::Hydra ]
+    attr_reader :hydra
+
+    # Create an easy factory.
+    #
+    # @example Create easy factory.
+    #   Typhoeus::Hydra::EasyFactory.new(request, hydra)
+    #
+    # @param [ Request ] request The request to build an easy for.
+    # @param [ Hydra ] hydra The hydra to build an easy for.
+    def initialize(request, hydra = nil)
+      @request = request
+      @hydra = hydra
+    end
+
+    # Return the easy in question.
+    #
+    # @example Return easy.
+    #   easy_factory.easy
+    #
+    # @return [ Ethon::Easy ] The easy.
+    def easy
+      @easy ||= Typhoeus::Pool.get
+    end
+
+    # Fabricated easy.
+    #
+    # @example Prepared easy.
+    #   easy_factory.get
+    #
+    # @return [ Ethon::Easy ] The easy.
+    def get
+      begin
+        easy.http_request(
+          request.base_url.to_s,
+          request.options.fetch(:method, :get),
+          sanitize(request.options)
+        )
+      rescue Ethon::Errors::InvalidOption => e
+        help = provide_help(e.message.match(/:\s(\w+)/)[1])
+        raise $!, "#{$!}#{help}", $!.backtrace
+      end
+      set_callback
+      easy
+    end
+
+    private
+
+    def sanitize(options)
+      # set nosignal to true by default
+      # this improves thread safety and timeout behavior
+      sanitized = {:nosignal => true}
+      options.each do |k,v|
+        s = k.to_sym
+        next if SANITIZE_IGNORE.include?(s)
+        if new_option = RENAMED_OPTIONS[k.to_sym]
+          warn("Deprecated option #{k}. Please use #{new_option} instead.")
+          sanitized[new_option] = v
+        # sanitize timeouts
+        elsif SANITIZE_TIMEOUT.include?(s)
+          if !v.integer?
+            warn("Value '#{v}' for option '#{k}' must be integer.")
+          end
+          sanitized[k] = v.ceil
+        else
+          sanitized[k] = v
+        end
+      end
+
+      sanitize_timeout!(sanitized, :timeout)
+      sanitize_timeout!(sanitized, :connecttimeout)
+
+      sanitized
+    end
+
+    def sanitize_timeout!(options, timeout)
+      timeout_ms = :"#{timeout}_ms"
+      if options[timeout] && options[timeout].round != options[timeout]
+        if !options[timeout_ms]
+          options[timeout_ms] = (options[timeout]*1000).ceil
+        end
+        options[timeout] = options[timeout].ceil
+      end
+      options
+    end
+
+    # Sets on_complete callback on easy in order to be able to
+    # track progress.
+    #
+    # @example Set callback.
+    #   easy_factory.set_callback
+    #
+    # @return [ Ethon::Easy ] The easy.
+    def set_callback
+      if request.streaming?
+        response = nil
+        easy.on_headers do |easy|
+          response = Response.new(Ethon::Easy::Mirror.from_easy(easy).options)
+          request.execute_headers_callbacks(response)
+        end
+        request.on_body.each do |callback|
+          easy.on_body do |chunk, easy|
+            callback.call(chunk, response)
+          end
+        end
+      else
+        easy.on_headers do |easy|
+          request.execute_headers_callbacks(Response.new(Ethon::Easy::Mirror.from_easy(easy).options))
+        end
+      end
+      request.on_progress.each do |callback|
+        easy.on_progress do |dltotal, dlnow, ultotal, ulnow, easy|
+          callback.call(dltotal, dlnow, ultotal, ulnow, response)
+        end
+      end
+      easy.on_complete do |easy|
+        request.finish(Response.new(easy.mirror.options))
+        Typhoeus::Pool.release(easy)
+        if hydra && !hydra.queued_requests.empty?
+          hydra.dequeue_many
+        end
+      end
+    end
+
+    def provide_help(option)
+      if new_option = CHANGED_OPTIONS[option.to_sym]
+        "\nPlease try #{new_option} instead of #{option}." if new_option
+      elsif REMOVED_OPTIONS.include?(option.to_sym)
+        "\nThe option #{option} was removed."
+      end
+    end
+  end
+end

data/lib/typhoeus/errors/no_stub.rb

@@ -0,0 +1,12 @@
+module Typhoeus
+  module Errors
+
+    # Raises when block connection is turned on
+    # and making a real request.
+    class NoStub < TyphoeusError
+      def initialize(request)
+        super("The connection is blocked and no stub defined: #{request.url}")
+      end
+    end
+  end
+end

data/lib/typhoeus/errors/typhoeus_error.rb

@@ -0,0 +1,8 @@
+module Typhoeus
+  module Errors
+
+    # Default typhoeus error class for all custom errors.
+    class TyphoeusError < StandardError
+    end
+  end
+end

data/lib/typhoeus/errors.rb

@@ -0,0 +1,9 @@
+require 'typhoeus/errors/typhoeus_error'
+require 'typhoeus/errors/no_stub'
+
+module Typhoeus
+
+  # This namespace contains all errors raised by Typhoeus.
+  module Errors
+  end
+end

data/lib/typhoeus/expectation.rb

@@ -0,0 +1,217 @@
+module Typhoeus
+
+  # This class represents an expectation. It is part
+  # of the stubbing mechanism. An expectation contains
+  # a url and options, like a request. They are compared
+  # to the request url and options in order to evaluate
+  # whether they match. If that's the case, the attached
+  # responses are returned one by one.
+  #
+  # @example Stub a request and get specified response.
+  #   expected = Typhoeus::Response.new
+  #   Typhoeus.stub("www.example.com").and_return(expected)
+  #
+  #   actual = Typhoeus.get("www.example.com")
+  #   expected == actual
+  #   #=> true
+  #
+  # @example Stub a request and get a lazily-constructed response containing data from actual widgets that exist in the system when the stubbed request is made.
+  #   Typhoeus.stub("www.example.com/widgets") do
+  #     actual_widgets = Widget.all
+  #     Typhoeus::Response.new(
+  #       :body => actual_widgets.inject([]) do |ids, widget|
+  #         ids << widget.id
+  #       end.join(",")
+  #     )
+  #   end
+  #
+  # @example Stub a request and get a lazily-constructed response in the format requested.
+  #   Typhoeus.stub("www.example.com") do |request|
+  #     accept = (request.options[:headers]||{})['Accept'] || "application/json"
+  #     format = accept.split(",").first
+  #     body_obj = { 'things' => [ { 'id' => 'foo' } ] }
+  #
+  #     Typhoeus::Response.new(
+  #       :headers => {
+  #         'Content-Type' => format
+  #       },
+  #       :body => SERIALIZERS[format].serialize(body_obj)
+  #     )
+  #   end
+  class Expectation
+
+    # @api private
+    attr_reader :base_url
+
+    # @api private
+    attr_reader :options
+
+    # @api private
+    attr_reader :from
+
+    class << self
+
+      # Returns all expectations.
+      #
+      # @example Return expectations.
+      #   Typhoeus::Expectation.all
+      #
+      # @return [ Array<Typhoeus::Expectation> ] The expectations.
+      def all
+        @expectations ||= []
+      end
+
+      # Clears expectations. This is handy while
+      # testing, and you want to make sure that
+      # you don't get canned responses.
+      #
+      # @example Clear expectations.
+      #   Typhoeus::Expectation.clear
+      def clear
+        all.clear
+      end
+
+      # Returns stubbed response matching the
+      # provided request.
+      #
+      # @example Find response
+      #   Typhoeus::Expectation.response_for(request)
+      #
+      # @return [ Typhoeus::Response ] The stubbed response from a
+      #   matching expectation, or nil if no matching expectation
+      #   is found.
+      #
+      # @api private
+      def response_for(request)
+        expectation = find_by(request)
+        return nil if expectation.nil?
+
+        expectation.response(request)
+      end
+
+      # @api private
+      def find_by(request)
+        all.find do |expectation|
+          expectation.matches?(request)
+        end
+      end
+    end
+
+    # Creates an expectation.
+    #
+    # @example Create expectation.
+    #   Typhoeus::Expectation.new(base_url)
+    #
+    # @return [ Expectation ] The created expectation.
+    #
+    # @api private
+    def initialize(base_url, options = {})
+      @base_url = base_url
+      @options = options
+      @response_counter = 0
+      @from = nil
+    end
+
+    # Set from value to mark an expectaion. Useful for
+    # other libraries, e.g. WebMock.
+    #
+    # @example Mark expectation.
+    #   expectation.from(:webmock)
+    #
+    # @param [ String ] value Value to set.
+    #
+    # @return [ Expectation ] Returns self.
+    #
+    # @api private
+    def stubbed_from(value)
+      @from = value
+      self
+    end
+
+    # Specify what should be returned,
+    # when this expectation is hit.
+    #
+    # @example Add response.
+    #   expectation.and_return(response)
+    #
+    # @return [ void ]
+    def and_return(response=nil, &block)
+      new_response = (response.nil? ? block : response)
+      responses.push(*new_response)
+    end
+
+    # Checks whether this expectation matches
+    # the provided request.
+    #
+    # @example Check if request matches.
+    #   expectation.matches? request
+    #
+    # @param [ Request ] request The request to check.
+    #
+    # @return [ Boolean ] True when matches, else false.
+    #
+    # @api private
+    def matches?(request)
+      url_match?(request.base_url) && options_match?(request)
+    end
+
+    # Return canned responses.
+    #
+    # @example Return responses.
+    #   expectation.responses
+    #
+    # @return [ Array<Typhoeus::Response> ] The responses.
+    #
+    # @api private
+    def responses
+      @responses ||= []
+    end
+
+    # Return the response. When there are
+    # multiple responses, they are returned one
+    # by one.
+    #
+    # @example Return response.
+    #   expectation.response
+    #
+    # @return [ Response ] The response.
+    #
+    # @api private
+    def response(request)
+      response = responses.fetch(@response_counter, responses.last)
+      if response.respond_to?(:call)
+        response = response.call(request)
+      end
+      @response_counter += 1
+      response.mock = @from || true
+      response
+    end
+
+    private
+
+    # Check whether the options matches the request options.
+    # I checks options and original options.
+    def options_match?(request)
+      (options ? options.all?{ |k,v| request.original_options[k] == v || request.options[k] == v } : true)
+    end
+
+    # Check whether the base_url matches the request url.
+    # The base_url can be a string, regex or nil. String and
+    # regexp are checked, nil is always true, else false.
+    #
+    # Nil serves as a placeholder in case you want to match
+    # all urls.
+    def url_match?(request_url)
+      case base_url
+      when String
+        base_url == request_url
+      when Regexp
+        base_url === request_url
+      when nil
+        true
+      else
+        false
+      end
+    end
+  end
+end

data/lib/typhoeus/hydra/addable.rb

@@ -0,0 +1,23 @@
+module Typhoeus
+  class Hydra
+
+    # This module handles the request adding on
+    # hydra.
+    #
+    # @api private
+    module Addable
+
+      # Adds request to multi.
+      #
+      # @example Add request.
+      #   hydra.add(request)
+      #
+      # @param [ Typhoeus::Request ] request to add.
+      #
+      # @return [ void ]
+      def add(request)
+        multi.add(EasyFactory.new(request, self).get)
+      end
+    end
+  end
+end

data/lib/typhoeus/hydra/before.rb

@@ -0,0 +1,31 @@
+module Typhoeus
+  class Hydra
+
+    # This module provides a way to hook into before
+    # a request gets queued in hydra. This is very powerful
+    # and you should be careful because when you accidently
+    # return a falsy value the request won't be executed.
+    #
+    # @api private
+    module Before
+
+      # Overrride add in order to execute callbacks in
+      # Typhoeus.before. Will break and return when a
+      # callback returns nil, false or a response. Calls super
+      # otherwise.
+      #
+      # @example Add the request.
+      #   hydra.add(request)
+      def add(request)
+        Typhoeus.before.each do |callback|
+          value = callback.call(request)
+          if value.nil? || value == false || value.is_a?(Response)
+            dequeue
+            return value
+          end
+        end
+        super
+      end
+    end
+  end
+end

data/lib/typhoeus/hydra/block_connection.rb

@@ -0,0 +1,35 @@
+module Typhoeus
+  class Hydra
+
+    # This module handles the blocked connection request mode on
+    # the hydra side, where only stubbed requests
+    # are allowed.
+    # Connection blocking needs to be turned on:
+    #   Typhoeus.configure do |config|
+    #     config.block_connection = true
+    #   end
+    #
+    # When trying to do real requests a NoStub error
+    # is raised.
+    #
+    # @api private
+    module BlockConnection
+
+      # Overrides add in order to check before if block connection
+      # is turned on. If thats the case a NoStub error is
+      # raised.
+      #
+      # @example Add the request.
+      #   hydra.add(request)
+      #
+      # @param [ Request ] request The request to enqueue.
+      def add(request)
+        if request.blocked?
+          raise Typhoeus::Errors::NoStub.new(request)
+        else
+          super
+        end
+      end
+    end
+  end
+end

data/lib/typhoeus/hydra/cacheable.rb

@@ -0,0 +1,15 @@
+module Typhoeus
+  class Hydra
+    module Cacheable
+      def add(request)
+        if request.cacheable? && response = request.cached_response
+          response.cached = true
+          request.finish(response)
+          dequeue
+        else
+          super
+        end
+      end
+    end
+  end
+end

data/lib/typhoeus/hydra/memoizable.rb

@@ -0,0 +1,56 @@
+module Typhoeus
+  class Hydra
+
+    # This module handles the GET request memoization
+    # on the hydra side. Memoization needs to be turned
+    # on:
+    #   Typhoeus.configure do |config|
+    #     config.memoize = true
+    #   end
+    #
+    # @api private
+    module Memoizable
+
+      # Return the memory.
+      #
+      # @example Return the memory.
+      #   hydra.memory
+      #
+      # @return [ Hash ] The memory.
+      def memory
+        @memory ||= {}
+      end
+
+      # Overrides add in order to check before if request
+      # is memoizable and already in memory. If thats the case,
+      # super is not called, instead the response is set and
+      # the on_complete callback called.
+      #
+      # @example Add the request.
+      #   hydra.add(request)
+      #
+      # @param [ Request ] request The request to add.
+      #
+      # @return [ Request ] The added request.
+      def add(request)
+        if request.memoizable? && memory.has_key?(request)
+          response = memory[request]
+          request.finish(response, true)
+          dequeue
+        else
+          super
+        end
+      end
+
+      # Overrides run to make sure the memory is cleared after
+      # each run.
+      #
+      # @example Run hydra.
+      #   hydra.run
+      def run
+        super
+        memory.clear
+      end
+    end
+  end
+end

data/lib/typhoeus/hydra/queueable.rb

@@ -0,0 +1,83 @@
+module Typhoeus
+  class Hydra
+
+    # This module handles the request queueing on
+    # hydra.
+    #
+    # @api private
+    module Queueable
+
+      # Return the queued requests.
+      #
+      # @example Return queued requests.
+      #   hydra.queued_requests
+      #
+      # @return [ Array<Typhoeus::Request> ] The queued requests.
+      def queued_requests
+        @queued_requests ||= []
+      end
+
+      # Abort the current hydra run as good as
+      # possible. This means that it only
+      # clears the queued requests and can't do
+      # anything about already running requests.
+      #
+      # @example Abort hydra.
+      #   hydra.abort
+      def abort
+        queued_requests.clear
+      end
+
+      # Enqueues a request in order to be performed
+      # by the hydra. This can even be done while
+      # the hydra is running. Also sets hydra on
+      # request.
+      #
+      # @example Queue request.
+      #   hydra.queue(request)
+      def queue(request)
+        request.hydra = self
+        queued_requests << request
+      end
+
+      # Pushes a request to the front of the queue,
+      # to be performed by the hydra. Also sets hydra
+      # on request
+      #
+      # @example Queue reques.
+      #   hydra.queue_front(request)
+      def queue_front(request)
+        request.hydra = self
+        queued_requests.unshift request
+      end
+
+      # Removes a request from queued_requests and
+      # adds it to the hydra in order to be
+      # performed next.
+      #
+      # @example Dequeue request.
+      #   hydra.dequeue
+      #
+      # @since 0.6.4
+      def dequeue
+        add(queued_requests.shift) unless queued_requests.empty?
+      end
+
+      # Removes requests from queued_requests and
+      # adds them to the hydra until max_concurrency
+      # is reached.
+      #
+      # @example Dequeue requests.
+      #   hydra.dequeue_many
+      #
+      # @since 0.6.8
+      def dequeue_many
+        number = multi.easy_handles.count
+        until number == max_concurrency || queued_requests.empty?
+          add(queued_requests.shift)
+          number += 1
+        end
+      end
+    end
+  end
+end

data/lib/typhoeus/hydra/runnable.rb

@@ -0,0 +1,19 @@
+module Typhoeus
+  class Hydra
+
+    # This module contains logic to run a hydra.
+    module Runnable
+
+      # Start the hydra run.
+      #
+      # @example Start hydra run.
+      #   hydra.run
+      #
+      # @return [ Symbol ] Return value from multi.perform.
+      def run
+        dequeue_many
+        multi.perform
+      end
+    end
+  end
+end