typhoeus 0.4.2 → 0.5.0.alpha

data/Rakefile

@@ -1,13 +1,18 @@
-$LOAD_PATH.unshift(File.dirname(__FILE__))
+require "bundler"
+Bundler.setup
 
-require 'rspec/core/rake_task'
-RSpec::Core::RakeTask.new do |t|
+require "rake"
+require "rspec/core/rake_task"
+$LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
+require "typhoeus/version"
+
+task :gem => :build
+task :build do
+  system "gem build typhoeus.gemspec"
 end
 
-task :install do
-  rm_rf "*.gem"
-  puts `gem build typhoeus.gemspec`
-  puts `gem install typhoeus-*.gem`
+task :install => :build do
+  system "gem install typhoeus-#{Typhoeus::VERSION}.gem"
 end
 
 task :release => :build do
@@ -16,14 +21,18 @@ task :release => :build do
   system "gem push typhoeus-#{Typhoeus::VERSION}.gem"
 end
 
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.verbose = false
+  t.ruby_opts = "-W -I./spec -rspec_helper"
+end
+
 desc "Start up the test servers"
-task :start_test_servers do
-  require 'spec/support/typhoeus_localhost_server'
+task :start do
+  require_relative 'spec/support/boot'
   begin
-    TyphoeusLocalhostServer.start_servers!(:rake)
+    Boot.start_servers(:rake)
   rescue Exception
   end
 end
 
-desc "Build Typhoeus and run all the tests."
-task :default => [:spec]
+task :default => :spec

data/lib/typhoeus.rb

@@ -1,56 +1,73 @@
 require 'digest/sha2'
+require 'ethon'
 
-require 'typhoeus/utils'
-require 'typhoeus/header'
-require 'typhoeus/curl'
-require 'typhoeus/easy'
-require 'typhoeus/form'
-require 'typhoeus/multi'
-require 'typhoeus/filter'
-require 'typhoeus/param_processor'
-require 'typhoeus/remote'
-require 'typhoeus/remote_proxy_object'
-require 'typhoeus/response'
-require 'typhoeus/request'
+require 'typhoeus/config'
+require 'typhoeus/errors'
+require 'typhoeus/expectation'
 require 'typhoeus/hydra'
-require 'typhoeus/hydra_mock'
+require 'typhoeus/request'
+require 'typhoeus/response'
 require 'typhoeus/version'
 
+# Typhoeus is a http client library based on Ethon which
+# wraps libcurl.
+#
+# If you want to make a single request, go with:
+#   Typhoeus.get("www.example.com")
+#
+# When you looking for firing a bunch of requests automatically
+# choose the hydra:
+#
+#   hydra = Typhoeus::Hydra.new
+#   requests = (0..9).map{ Typhoeus::Request.new("www.example.com") }
+#   requests.each{ |request| hydra.queue(request) }
+#   hydra.run
 module Typhoeus
-  def self.easy_object_pool
-    @easy_objects ||= []
-  end
+  extend self
+  extend Hydras::EasyPool
+  extend Requests::Actions
+  extend Requests::Callbacks::Types
 
-  def self.init_easy_object_pool
-    20.times do
-      easy_object_pool << Typhoeus::Easy.new
-    end
-  end
+  # The default typhoeus user agent.
+  USER_AGENT = "Typhoeus - https://github.com/typhoeus/typhoeus"
 
-  def self.release_easy_object(easy)
-    easy.reset
-    easy_object_pool << easy
+  # Set the Typhoeus configuration options by passing a block.
+  #
+  # @example Set the configuration options.
+  #   Typhoeus.configure do |config|
+  #     config.verbose = true
+  #   end
+  #
+  # @return [ Config ] The configuration object.
+  def configure
+    yield Config
   end
 
-  def self.get_easy_object
-    if easy_object_pool.empty?
-      Typhoeus::Easy.new
-    else
-      easy_object_pool.pop
-    end
-  end
+  # Stub out specific request.
+  #
+  # @example Stub.
+  #   Typhoeus.stub("www.example.com").and_return(Typhoeus::Response.new)
+  #
+  # @param [ String ] url The url to stub out.
+  # @param [ Hash ] options The options to stub out.
+  #
+  # @return [ Expection ] The expection.
+  def stub(url, options = {})
+    expectation = Expectation.all.find{ |e| e.url == url && e.options == options }
+    return expectation if expectation
 
-  def self.add_easy_request(easy_object)
-    Thread.current[:curl_multi] ||= Typhoeus::Multi.new
-    Thread.current[:curl_multi].add(easy_object)
+    Expectation.new(url, options).tap do |new_expectation|
+      Expectation.all << new_expectation
+    end
   end
 
-  def self.perform_easy_requests
-    multi = Thread.current[:curl_multi]
-    start_time = Time.now
-    multi.easy_handles.each do |easy|
-      easy.start_time = start_time
-    end
-    multi.perform
+  # Execute given block as if block connection is turned off.
+  #
+  # @param [ Block ] block The block to execute.
+  def with_connection
+    old = Config.block_connection
+    Config.block_connection = false
+    yield if block_given?
+    Config.block_connection = old
   end
 end

data/lib/typhoeus/config.rb

@@ -0,0 +1,14 @@
+module Typhoeus
+
+  # The Typhoeus configuration used to set global
+  # options. Available options:
+  # * block_connection: only stubbed requests are
+  #   allowed, raises NoStub error when trying to
+  #   do a real request.
+  # * verbose: show curls debug out
+  # * memoize: memoize GET requests.
+  module Config
+    extend self
+    attr_accessor :block_connection, :memoize, :verbose
+  end
+end

data/lib/typhoeus/errors.rb

@@ -0,0 +1,9 @@
+require 'typhoeus/errors/typhoeus_error'
+require 'typhoeus/errors/no_stub'
+
+module Typhoeus
+
+  # This namespace contains all errors raised by typhoeus.
+  module Errors
+  end
+end

data/lib/typhoeus/errors/no_stub.rb

@@ -0,0 +1,12 @@
+module Typhoeus
+  module Errors
+
+    # Raises when block connection is turned on
+    # and making a real request.
+    class NoStub < TyphoeusError
+      def initialize(request)
+        super("The connection is blocked and no stub defined.")
+      end
+    end
+  end
+end

data/lib/typhoeus/errors/typhoeus_error.rb

@@ -0,0 +1,8 @@
+module Typhoeus
+  module Errors
+
+    # Default typhoeus error class for all custom errors.
+    class TyphoeusError < StandardError
+    end
+  end
+end

data/lib/typhoeus/expectation.rb

@@ -0,0 +1,126 @@
+module Typhoeus
+
+  # This class represents an expectation. It is part
+  # of the stubbing mechanism. An expectation contains
+  # an url and options, like a request. They were compared
+  # to the request url and options in order to evaluate
+  # wether they match. If thats the case, the attached
+  # responses were returned one by one.
+  class Expectation
+    attr_reader :url, :options
+
+    class << self
+
+      # Returns all expectations.
+      #
+      # @example Return expectations.
+      #   Typhoeus::Expectation.all
+      #
+      # @return [ Array ] The expectations.
+      def all
+        @expectations ||= []
+      end
+
+      # Clears expectations.
+      #
+      # @example Clear expectations.
+      #   Typhoeus:;Expectation.clear
+      def clear
+        all.clear
+      end
+
+      # Returns expecation matching the provided
+      # request.
+      #
+      # @example Find expectation.
+      #   Typhoeus::Expectation.find_by(request)
+      #
+      # @return [ Expectation ] The matching expectation.
+      def find_by(request)
+        all.find do |expectation|
+          expectation.matches?(request)
+        end
+      end
+    end
+
+    # Creates an expactation.
+    #
+    # @example Create expactation.
+    #   Typhoeus::Expectation.new(url)
+    #
+    # @return [ Expectation ] The created expactation.
+    def initialize(url, options = {})
+      @url = url
+      @options = options
+      @response_counter = 0
+    end
+
+    # Specify what should be returned,
+    # when this expactation is hit.
+    #
+    # @example Add response.
+    #   expectation.and_return(response)
+    def and_return(response)
+      responses << response
+    end
+
+    # Checks wether this expectation matches
+    # the provided request.
+    #
+    # @example Check if request matches.
+    #   expectation.matches? request
+    #
+    # @param [ Request ] The request to check.
+    #
+    # @return [ Boolean ] True when matches, else false.
+    def matches?(request)
+      url_match?(request.url) &&
+        (options ? options.all?{ |k,v| request.original_options[k] == v } : true)
+    end
+
+    # Return canned responses.
+    #
+    # @example Return responses.
+    #   expectation.responses
+    #
+    # @return [ Array ] The responses.
+    def responses
+      @responses ||= []
+    end
+
+    # Return the response. When there are
+    # multiple responses, they were returned one
+    # by one.
+    #
+    # @example Return response.
+    #   expectation.response
+    #
+    # @return [ Response ] The response.
+    def response
+      response = responses.fetch(@response_counter, responses.last)
+      @response_counter += 1
+      response
+    end
+
+    private
+
+    # Check wether the url matches the request url.
+    # The url can be a string, regex or nil. String and
+    # regexp were checked, nil is always true. Else false.
+    #
+    # Nil serves as a placeholder in case you want to match
+    # all urls.
+    def url_match?(request_url)
+      case url
+      when String
+        url == request_url
+      when Regexp
+        !!request_url.match(url)
+      when nil
+        true
+      else
+        false
+      end
+    end
+  end
+end

data/lib/typhoeus/hydra.rb

@@ -1,247 +1,42 @@
-require 'typhoeus/hydra/callbacks'
-require 'typhoeus/hydra/connect_options'
-require 'typhoeus/hydra/stubbing'
+require 'typhoeus/hydras/block_connection'
+require 'typhoeus/hydras/easy_factory'
+require 'typhoeus/hydras/easy_pool'
+require 'typhoeus/hydras/memoizable'
+require 'typhoeus/hydras/queueable'
+require 'typhoeus/hydras/runnable'
+require 'typhoeus/hydras/stubbable'
 
 module Typhoeus
-  class Hydra
-    include ConnectOptions
-    include Stubbing
-    extend Callbacks
-
-    def initialize(options = {})
-      @memoize_requests = true
-      @multi       = Multi.new
-      @easy_pool   = []
-      initial_pool_size = options[:initial_pool_size] || 10
-      @max_concurrency = options[:max_concurrency] || 200
-      initial_pool_size.times { @easy_pool << Easy.new }
-      @memoized_requests = {}
-      @retrieved_from_cache = {}
-      @queued_requests = []
-      @running_requests = 0
 
-      self.stubs = []
-      @active_stubs = []
-    end
+  # Hydra manages making parallel HTTP requests. This
+  # is achieved by using libcurls multi interface:
+  # http://curl.haxx.se/libcurl/c/libcurl-multi.html
+  # The benefits are that you don't have to worry running
+  # the requests by yourself.
+  class Hydra
+    include Hydras::EasyPool
+    include Hydras::Queueable
+    include Hydras::Runnable
+    include Hydras::Memoizable
+    include Hydras::BlockConnection
+    include Hydras::Stubbable
 
-    def self.hydra
-      @hydra ||= new
-    end
-
-    def self.hydra=(val)
-      @hydra = val
-    end
+    attr_reader :max_concurrency, :multi
 
+    # Create a new hydra.
     #
-    # Abort the run on a best-effort basis.
+    # @example Create a hydra.
+    #   Typhoeus::Hydra.new
     #
-    # It won't abort the current burst of @max_concurrency requests,
-    # however it won't fire the rest of the queued requests so the run
-    # will be aborted as soon as possible...
+    # @param [ Hash ] options The options hash.
     #
-    def abort
-      @queued_requests.clear
-    end
-
-    def clear_cache_callbacks
-      @cache_setter = nil
-      @cache_getter = nil
-    end
-
-    def fire_and_forget
-      @queued_requests.each {|r| queue(r, false)}
-      @multi.fire_and_forget
-    end
-
-    def queue(request, obey_concurrency_limit = true)
-      return if assign_to_stub(request)
-
-      # At this point, we are running over live HTTP. Make sure we haven't
-      # disabled live requests.
-      check_allow_net_connect!(request)
-
-      if @running_requests >= @max_concurrency && obey_concurrency_limit
-        @queued_requests << request
-      else
-        if request.method == :get
-          if @memoize_requests && @memoized_requests.key?(request.url)
-            if response = @retrieved_from_cache[request.url]
-              request.response = response
-              request.call_handlers
-            else
-              @memoized_requests[request.url] << request
-            end
-          else
-            @memoized_requests[request.url] = [] if @memoize_requests
-            get_from_cache_or_queue(request)
-          end
-        else
-          get_from_cache_or_queue(request)
-        end
-      end
-    end
-
-    def run
-      while !@active_stubs.empty?
-        m = @active_stubs.first
-        while request = m.requests.shift
-          response = m.response
-          response.request = request
-          handle_request(request, response)
-        end
-        @active_stubs.delete(m)
-      end
-
-      @multi.perform
-    ensure
-      @multi.reset_easy_handles{|easy| release_easy_object(easy)}
-      @memoized_requests = {}
-      @retrieved_from_cache = {}
-      @running_requests = 0
-    end
-
-    def disable_memoization
-      @memoize_requests = false
-    end
-
-    def cache_getter(&block)
-      @cache_getter = block
-    end
-
-    def cache_setter(&block)
-      @cache_setter = block
-    end
-
-    def on_complete(&block)
-      @on_complete = block
-    end
-
-    def on_complete=(proc)
-      @on_complete = proc
-    end
-
-    private
-
-    def get_from_cache_or_queue(request)
-      if @cache_getter
-        val = @cache_getter.call(request)
-        if val
-          @retrieved_from_cache[request.url] = val
-          queue_next
-          handle_request(request, val, false)
-        else
-          @multi.add(get_easy_object(request))
-        end
-      else
-        @multi.add(get_easy_object(request))
-      end
-    end
-
-    def get_easy_object(request)
-      @running_requests += 1
-
-      easy = @easy_pool.pop || Easy.new
-      easy.verbose          = request.verbose
-      if request.username || request.password
-        auth = { :username => request.username, :password => request.password }
-        auth[:method] = request.auth_method if request.auth_method
-        easy.auth = auth
-      end
-
-      if request.proxy
-        proxy = { :server => request.proxy }
-        proxy[:type] = request.proxy_type if request.proxy_type
-        easy.proxy = proxy if request.proxy
-      end
-
-      if request.proxy_username || request.proxy_password
-        auth = { :username => request.proxy_username, :password => request.proxy_password }
-        auth[:method] = request.proxy_auth_method if request.proxy_auth_method
-        easy.proxy_auth = auth
-      end
-
-      easy.url          = request.url
-      easy.method       = request.method
-      easy.params       = request.params  if [:post, :put].include?(request.method) && !request.params.nil?
-      easy.headers      = request.headers if request.headers
-      easy.request_body = request.body    if [:post, :put].include?(request.method) && !request.body.nil?
-      easy.timeout      = request.timeout if request.timeout
-      easy.connect_timeout = request.connect_timeout if request.connect_timeout
-      easy.interface       = request.interface if request.interface
-      easy.follow_location = request.follow_location if request.follow_location
-      easy.max_redirects = request.max_redirects if request.max_redirects
-      easy.disable_ssl_peer_verification if request.disable_ssl_peer_verification
-      easy.disable_ssl_host_verification if request.disable_ssl_host_verification
-      easy.ssl_cert         = request.ssl_cert
-      easy.ssl_cert_type    = request.ssl_cert_type
-      easy.ssl_key          = request.ssl_key
-      easy.ssl_key_type     = request.ssl_key_type
-      easy.ssl_key_password = request.ssl_key_password
-      easy.ssl_cacert       = request.ssl_cacert
-      easy.ssl_capath       = request.ssl_capath
-      easy.ssl_version      = request.ssl_version || :default
-      easy.verbose          = request.verbose
-
-      easy.on_success do |easy|
-        queue_next
-        handle_request(request, response_from_easy(easy, request))
-        release_easy_object(easy)
-      end
-      easy.on_failure do |easy|
-        queue_next
-        handle_request(request, response_from_easy(easy, request))
-        release_easy_object(easy)
-      end
-      easy.set_headers
-      easy
-    end
-
-    def queue_next
-      @running_requests -= 1
-      queue(@queued_requests.shift) unless @queued_requests.empty?
-    end
-
-    def release_easy_object(easy)
-      easy.reset
-      @easy_pool.push easy
-    end
-
-    def handle_request(request, response, live_request = true)
-      request.response = response
-
-      self.class.run_global_hooks_for(:after_request_before_on_complete,
-                                      request)
-
-      if live_request && request.cache_timeout && @cache_setter
-        @cache_setter.call(request)
-      end
-      @on_complete.call(response) if @on_complete
-
-      request.call_handlers
-      if requests = @memoized_requests[request.url]
-        requests.each do |r|
-          r.response = response
-          r.call_handlers
-        end
-      end
-    end
-
-    def response_from_easy(easy, request)
-      Response.new(:code                => easy.response_code,
-                   :headers             => easy.response_header,
-                   :body                => easy.response_body,
-                   :time                => easy.total_time_taken,
-                   :start_transfer_time => easy.start_transfer_time,
-                   :app_connect_time    => easy.app_connect_time,
-                   :pretransfer_time    => easy.pretransfer_time,
-                   :connect_time        => easy.connect_time,
-                   :name_lookup_time    => easy.name_lookup_time,
-                   :effective_url       => easy.effective_url,
-                   :primary_ip          => easy.primary_ip,
-                   :curl_return_code => easy.curl_return_code,
-                   :curl_error_message => easy.curl_error_message,
-                   :redirect_count      => easy.redirect_count,
-                   :request             => request)
+    # @option options :max_concurrency [ Integer ] Number
+    #  of max concurrent connections to create. Default is
+    #  200.
+    def initialize(options = {})
+      @options = options
+      @max_concurrency = @options.fetch(:max_concurrency, 200)
+      @multi = Ethon::Multi.new
     end
   end
 end