typhoeus 0.4.2 → 0.5.0.pre

data/lib/typhoeus/hydra.rb

@@ -1,247 +1,37 @@
-require 'typhoeus/hydra/callbacks'
-require 'typhoeus/hydra/connect_options'
-require 'typhoeus/hydra/stubbing'
+require 'typhoeus/hydras/easy_factory'
+require 'typhoeus/hydras/easy_pool'
+require 'typhoeus/hydras/memoizable'
+require 'typhoeus/hydras/queueable'
+require 'typhoeus/hydras/runnable'
 
 module Typhoeus
-  class Hydra
-    include ConnectOptions
-    include Stubbing
-    extend Callbacks
-
-    def initialize(options = {})
-      @memoize_requests = true
-      @multi       = Multi.new
-      @easy_pool   = []
-      initial_pool_size = options[:initial_pool_size] || 10
-      @max_concurrency = options[:max_concurrency] || 200
-      initial_pool_size.times { @easy_pool << Easy.new }
-      @memoized_requests = {}
-      @retrieved_from_cache = {}
-      @queued_requests = []
-      @running_requests = 0
 
-      self.stubs = []
-      @active_stubs = []
-    end
+  # Hydra manages making parallel HTTP requests. This
+  # is archived by using libcurls multi interface. The
+  # benefits are that you don't have to worry running
+  # the requests by yourself.
+  class Hydra
+    include Hydras::EasyPool
+    include Hydras::Queueable
+    include Hydras::Runnable
+    include Hydras::Memoizable
 
-    def self.hydra
-      @hydra ||= new
-    end
-
-    def self.hydra=(val)
-      @hydra = val
-    end
+    attr_reader :max_concurrency, :multi
 
+    # Create a new hydra.
     #
-    # Abort the run on a best-effort basis.
+    # @example Create a hydra.
+    #   Typhoeus::Hydra.new
     #
-    # It won't abort the current burst of @max_concurrency requests,
-    # however it won't fire the rest of the queued requests so the run
-    # will be aborted as soon as possible...
+    # @param [ Hash ] options The options hash.
     #
-    def abort
-      @queued_requests.clear
-    end
-
-    def clear_cache_callbacks
-      @cache_setter = nil
-      @cache_getter = nil
-    end
-
-    def fire_and_forget
-      @queued_requests.each {|r| queue(r, false)}
-      @multi.fire_and_forget
-    end
-
-    def queue(request, obey_concurrency_limit = true)
-      return if assign_to_stub(request)
-
-      # At this point, we are running over live HTTP. Make sure we haven't
-      # disabled live requests.
-      check_allow_net_connect!(request)
-
-      if @running_requests >= @max_concurrency && obey_concurrency_limit
-        @queued_requests << request
-      else
-        if request.method == :get
-          if @memoize_requests && @memoized_requests.key?(request.url)
-            if response = @retrieved_from_cache[request.url]
-              request.response = response
-              request.call_handlers
-            else
-              @memoized_requests[request.url] << request
-            end
-          else
-            @memoized_requests[request.url] = [] if @memoize_requests
-            get_from_cache_or_queue(request)
-          end
-        else
-          get_from_cache_or_queue(request)
-        end
-      end
-    end
-
-    def run
-      while !@active_stubs.empty?
-        m = @active_stubs.first
-        while request = m.requests.shift
-          response = m.response
-          response.request = request
-          handle_request(request, response)
-        end
-        @active_stubs.delete(m)
-      end
-
-      @multi.perform
-    ensure
-      @multi.reset_easy_handles{|easy| release_easy_object(easy)}
-      @memoized_requests = {}
-      @retrieved_from_cache = {}
-      @running_requests = 0
-    end
-
-    def disable_memoization
-      @memoize_requests = false
-    end
-
-    def cache_getter(&block)
-      @cache_getter = block
-    end
-
-    def cache_setter(&block)
-      @cache_setter = block
-    end
-
-    def on_complete(&block)
-      @on_complete = block
-    end
-
-    def on_complete=(proc)
-      @on_complete = proc
-    end
-
-    private
-
-    def get_from_cache_or_queue(request)
-      if @cache_getter
-        val = @cache_getter.call(request)
-        if val
-          @retrieved_from_cache[request.url] = val
-          queue_next
-          handle_request(request, val, false)
-        else
-          @multi.add(get_easy_object(request))
-        end
-      else
-        @multi.add(get_easy_object(request))
-      end
-    end
-
-    def get_easy_object(request)
-      @running_requests += 1
-
-      easy = @easy_pool.pop || Easy.new
-      easy.verbose          = request.verbose
-      if request.username || request.password
-        auth = { :username => request.username, :password => request.password }
-        auth[:method] = request.auth_method if request.auth_method
-        easy.auth = auth
-      end
-
-      if request.proxy
-        proxy = { :server => request.proxy }
-        proxy[:type] = request.proxy_type if request.proxy_type
-        easy.proxy = proxy if request.proxy
-      end
-
-      if request.proxy_username || request.proxy_password
-        auth = { :username => request.proxy_username, :password => request.proxy_password }
-        auth[:method] = request.proxy_auth_method if request.proxy_auth_method
-        easy.proxy_auth = auth
-      end
-
-      easy.url          = request.url
-      easy.method       = request.method
-      easy.params       = request.params  if [:post, :put].include?(request.method) && !request.params.nil?
-      easy.headers      = request.headers if request.headers
-      easy.request_body = request.body    if [:post, :put].include?(request.method) && !request.body.nil?
-      easy.timeout      = request.timeout if request.timeout
-      easy.connect_timeout = request.connect_timeout if request.connect_timeout
-      easy.interface       = request.interface if request.interface
-      easy.follow_location = request.follow_location if request.follow_location
-      easy.max_redirects = request.max_redirects if request.max_redirects
-      easy.disable_ssl_peer_verification if request.disable_ssl_peer_verification
-      easy.disable_ssl_host_verification if request.disable_ssl_host_verification
-      easy.ssl_cert         = request.ssl_cert
-      easy.ssl_cert_type    = request.ssl_cert_type
-      easy.ssl_key          = request.ssl_key
-      easy.ssl_key_type     = request.ssl_key_type
-      easy.ssl_key_password = request.ssl_key_password
-      easy.ssl_cacert       = request.ssl_cacert
-      easy.ssl_capath       = request.ssl_capath
-      easy.ssl_version      = request.ssl_version || :default
-      easy.verbose          = request.verbose
-
-      easy.on_success do |easy|
-        queue_next
-        handle_request(request, response_from_easy(easy, request))
-        release_easy_object(easy)
-      end
-      easy.on_failure do |easy|
-        queue_next
-        handle_request(request, response_from_easy(easy, request))
-        release_easy_object(easy)
-      end
-      easy.set_headers
-      easy
-    end
-
-    def queue_next
-      @running_requests -= 1
-      queue(@queued_requests.shift) unless @queued_requests.empty?
-    end
-
-    def release_easy_object(easy)
-      easy.reset
-      @easy_pool.push easy
-    end
-
-    def handle_request(request, response, live_request = true)
-      request.response = response
-
-      self.class.run_global_hooks_for(:after_request_before_on_complete,
-                                      request)
-
-      if live_request && request.cache_timeout && @cache_setter
-        @cache_setter.call(request)
-      end
-      @on_complete.call(response) if @on_complete
-
-      request.call_handlers
-      if requests = @memoized_requests[request.url]
-        requests.each do |r|
-          r.response = response
-          r.call_handlers
-        end
-      end
-    end
-
-    def response_from_easy(easy, request)
-      Response.new(:code                => easy.response_code,
-                   :headers             => easy.response_header,
-                   :body                => easy.response_body,
-                   :time                => easy.total_time_taken,
-                   :start_transfer_time => easy.start_transfer_time,
-                   :app_connect_time    => easy.app_connect_time,
-                   :pretransfer_time    => easy.pretransfer_time,
-                   :connect_time        => easy.connect_time,
-                   :name_lookup_time    => easy.name_lookup_time,
-                   :effective_url       => easy.effective_url,
-                   :primary_ip          => easy.primary_ip,
-                   :curl_return_code => easy.curl_return_code,
-                   :curl_error_message => easy.curl_error_message,
-                   :redirect_count      => easy.redirect_count,
-                   :request             => request)
+    # @option options :max_concurrency [ Integer ] Number
+    #  of max concurrent connections to create. Default is
+    #  200.
+    def initialize(options = {})
+      @options = options
+      @max_concurrency = @options.fetch(:max_concurrency, 200)
+      @multi = Ethon::Multi.new
     end
   end
 end

data/lib/typhoeus/hydras/easy_factory.rb

@@ -0,0 +1,67 @@
+module Typhoeus
+  module Hydras
+
+    # This is a Factory for easies to be used in the hydra.
+    # Before an easy is ready to be added to a multi, it needs
+    # to be prepared and the on_complete callback to be set.
+    # This is done by this class.
+    class EasyFactory
+      attr_reader :request, :hydra
+
+      # Create an easy factory.
+      #
+      # @example Create easy factory.
+      #   Typhoeus::Hydras::EasyFactory.new(request, hydra)
+      #
+      # @param [ Request ] request The request to build an easy for.
+      # @param [ Hydra ] hydra The hydra to build an easy for.
+      def initialize(request, hydra)
+        @request = request
+        @hydra = hydra
+      end
+
+      # Return the easy in question.
+      #
+      # @example Return easy.
+      #   easy_factory.easy
+      #
+      # @return [ Ethon::Easy ] The easy.
+      def easy
+        @easy ||= hydra.get_easy
+      end
+
+      # Fabricated and prepared easy.
+      #
+      # @example Prepared easy.
+      #   easy_factory.get
+      #
+      # @return [ Ethon::Easy ] The prepared easy.
+      def get
+        easy.http_request(
+          request.url,
+          request.options.fetch(:method, :get),
+          request.options.reject{|k,_| k==:method}
+        )
+        easy.prepare
+        set_callback
+        easy
+      end
+
+      # Sets on_complete callback on easy in order to be able to
+      # track progress.
+      #
+      # @example Set callback.
+      #   easy_factory.set_callback
+      #
+      # @return [ Ethon::Easy ] The easy.
+      def set_callback
+        easy.on_complete do |easy|
+          request.response = Response.new(easy.to_hash)
+          hydra.release_easy(easy)
+          hydra.queue(hydra.queued_requests.shift) unless hydra.queued_requests.empty?
+          request.complete
+        end
+      end
+    end
+  end
+end

data/lib/typhoeus/hydras/easy_pool.rb

@@ -0,0 +1,40 @@
+module Typhoeus
+  module Hydras
+
+    # The easy pool stores already initialized
+    # easy handles for future use. This is useful
+    # because creating them is quite expensive.
+    module EasyPool
+
+      # Return the easy pool.
+      #
+      # @example Return easy pool.
+      #   hydra.easy_pool
+      #
+      # @return [ Array ] The easy pool.
+      def easy_pool
+        @easy_pool ||= []
+      end
+
+      # Releases easy into pool. The easy handle is
+      # resetted before it gets back in.
+      #
+      # @example Release easy.
+      #   hydra.release_easy(easy)
+      def release_easy(easy)
+        easy.reset
+        easy_pool << easy
+      end
+
+      # Return an easy from pool.
+      #
+      # @example Return easy.
+      #   hydra.get_easy
+      #
+      # @return [ Ethon::Easy ] The easy.
+      def get_easy
+        easy_pool.pop || Ethon::Easy.new
+      end
+    end
+  end
+end

data/lib/typhoeus/hydras/memoizable.rb

@@ -0,0 +1,53 @@
+module Typhoeus
+  module Hydras
+
+    # This module handles the GET request memoization
+    # on the hydra side. Memoization needs to be turned
+    # on:
+    #   Typhoeus.configre do |config|
+    #     config.memoize = true
+    #   end
+    module Memoizable
+
+      # Return the memory.
+      #
+      # @example Return the memory.
+      #   hydra.memory
+      #
+      # @return [ Hash ] The memory.
+      def memory
+        @memory ||= {}
+      end
+
+      # Overrides queue in order to check before if request
+      # is memoizable and already in memory. If thats the case,
+      # super is not called, instead the response is set and
+      # the on_complete callback called.
+      #
+      # @example Queue the request.
+      #   hydra.queue(request)
+      #
+      # @param [ Request ] request The request to enqueue.
+      #
+      # @return [ Request ] The queued request.
+      def queue(request)
+        if request.memoizable? && memory.has_key?(request)
+          request.instance_variable_set(:@response, memory[request])
+          request.complete
+        else
+          super
+        end
+      end
+
+      # Overrides run to clear the memory is cleared after
+      # each run.
+      #
+      # @example Run hydra.
+      #   hydra.run
+      def run
+        super
+        memory.clear
+      end
+    end
+  end
+end

data/lib/typhoeus/hydras/queueable.rb

@@ -0,0 +1,46 @@
+module Typhoeus
+  module Hydras # :nodoc:
+
+    # This module handles the request queueing on
+    # hydra.
+    module Queueable
+
+      # Return the queued requests.
+      #
+      # @example Return queued requests.
+      #  hydra.queued_requests
+      #
+      # @return [ Array ] The queued requests.
+      def queued_requests
+        @queued_requests ||= []
+      end
+
+      # Abort the current hydra run as good as
+      # possible. This means that it only
+      # clears the queued requests and can't do
+      # anything about already running requests.
+      #
+      # @example Abort hydra.
+      #  hydra.abort
+      def abort
+        queued_requests.clear
+      end
+
+      # Enqueues a request in order to be performed
+      # by the hydra. This can even be done while
+      # the hydra is running. Also sets hydra on
+      # request.
+      #
+      # @example Queue request.
+      #  hydra.queue(request)
+      def queue(request)
+        request.hydra = self
+        if multi.easy_handles.size < max_concurrency
+          multi.add(Hydras::EasyFactory.new(request, self).get)
+        else
+          queued_requests << request
+        end
+      end
+    end
+  end
+end

data/lib/typhoeus/hydras/runnable.rb

@@ -0,0 +1,18 @@
+module Typhoeus
+  module Hydras
+
+    # This module contains logic to run a hydra.
+    module Runnable
+
+      # Start the hydra run.
+      #
+      # @example Start hydra run.
+      #   hydra.run
+      #
+      # @return [ Symbol ] Return value from multi.perform.
+      def run
+        multi.perform
+      end
+    end
+  end
+end