typhoeus 0.4.2 → 0.5.0.rc

data/lib/typhoeus/config.rb

@@ -0,0 +1,47 @@
+module Typhoeus
+
+  # The Typhoeus configuration used to set global
+  # options.
+  # @example Set the configuration options within a block.
+  #   Typhoeus.configure do |config|
+  #     config.verbose = true
+  #   end
+  #
+  # @example Set the configuration direct.
+  #   Typhoeus::Config.verbose = true
+  module Config
+    extend self
+
+    # Defines wether the connection is blocked.
+    # Defaults to false. When set to true, only
+    # stubbed requests are allowed. A
+    # {Typhoeus::Errors::NoStub} error is raised,
+    # when trying to do a real request. Its possible
+    # to work around inside
+    # {Typhoeus#with_connection}.
+    #
+    # @return [ Boolean ]
+    #
+    # @see Typhoeus::Request::BlockConnection
+    # @see Typhoeus::Hydra::BlockConnection
+    # @see Typhoeus#with_connection
+    # @see Typhoeus::Errors::NoStub
+    attr_accessor :block_connection
+
+    # Defines wether GET requests are memoized when using the {Typhoeus::Hydra}.
+    #
+    # @return [ Boolean ]
+    #
+    # @see Typhoeus::Hydra
+    # @see Typhoeus::Hydra::Memoizable
+    attr_accessor :memoize
+
+    # Defines wether curls debug output is shown.
+    # Unfortunately it prints to stderr.
+    #
+    # @return [ Boolean ]
+    #
+    # @see http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTVERBOSE
+    attr_accessor :verbose
+  end
+end

data/lib/typhoeus/errors/no_stub.rb

@@ -0,0 +1,12 @@
+module Typhoeus
+  module Errors
+
+    # Raises when block connection is turned on
+    # and making a real request.
+    class NoStub < TyphoeusError
+      def initialize(request)
+        super("The connection is blocked and no stub defined.")
+      end
+    end
+  end
+end

data/lib/typhoeus/errors/typhoeus_error.rb

@@ -0,0 +1,8 @@
+module Typhoeus
+  module Errors
+
+    # Default typhoeus error class for all custom errors.
+    class TyphoeusError < StandardError
+    end
+  end
+end

data/lib/typhoeus/errors.rb

@@ -0,0 +1,9 @@
+require 'typhoeus/errors/typhoeus_error'
+require 'typhoeus/errors/no_stub'
+
+module Typhoeus
+
+  # This namespace contains all errors raised by typhoeus.
+  module Errors
+  end
+end

data/lib/typhoeus/expectation.rb

@@ -0,0 +1,174 @@
+module Typhoeus
+
+  # This class represents an expectation. It is part
+  # of the stubbing mechanism. An expectation contains
+  # an url and options, like a request. They were compared
+  # to the request url and options in order to evaluate
+  # wether they match. If thats the case, the attached
+  # responses were returned one by one.
+  #
+  # @example Stub a request and get specified response.
+  #   expected = Typhoeus::Response.new
+  #   Typhoeus.stub("www.example.com").and_return(expected)
+  #
+  #   actual = Typhoeus.get("www.example.com")
+  #   expected == actual
+  #   #=> true
+  class Expectation
+
+    # @api private
+    attr_reader :url
+
+    # @api private
+    attr_reader :options
+
+    # @api private
+    attr_reader :from
+
+    class << self
+
+      # Returns all expectations.
+      #
+      # @example Return expectations.
+      #   Typhoeus::Expectation.all
+      #
+      # @return [ Array<Typhoeus::Expectation> ] The expectations.
+      def all
+        @expectations ||= []
+      end
+
+      # Clears expectations. This is handy while
+      # testing and you want to make sure, that
+      # you don't get canned responses.
+      #
+      # @example Clear expectations.
+      #   Typhoeus::Expectation.clear
+      def clear
+        all.clear
+      end
+
+      # Returns expecation matching the provided
+      # request.
+      #
+      # @example Find expectation.
+      #   Typhoeus::Expectation.find_by(request)
+      #
+      # @return [ Expectation ] The matching expectation.
+      #
+      # @api private
+      def find_by(request)
+        all.find do |expectation|
+          expectation.matches?(request)
+        end
+      end
+    end
+
+    # Creates an expactation.
+    #
+    # @example Create expactation.
+    #   Typhoeus::Expectation.new(url)
+    #
+    # @return [ Expectation ] The created expactation.
+    #
+    # @api private
+    def initialize(url, options = {})
+      @url = url
+      @options = options
+      @response_counter = 0
+      @from = nil
+    end
+
+    # Set from value to mark an expecation. Useful for
+    # other libraries, eg. webmock.
+    #
+    # @example Mark expecation.
+    #   expecation.from(:webmock)
+    #
+    # @param [ String ] value Value to set.
+    #
+    # @return [ Expectation ] Returns self.
+    #
+    # @api private
+    def stubbed_from(value)
+      @from = value
+      self
+    end
+
+    # Specify what should be returned,
+    # when this expactation is hit.
+    #
+    # @example Add response.
+    #   expectation.and_return(response)
+    #
+    # @return [ void ]
+    def and_return(response)
+      responses << response
+    end
+
+    # Checks wether this expectation matches
+    # the provided request.
+    #
+    # @example Check if request matches.
+    #   expectation.matches? request
+    #
+    # @param [ Request ] request The request to check.
+    #
+    # @return [ Boolean ] True when matches, else false.
+    #
+    # @api private
+    def matches?(request)
+      url_match?(request.url) &&
+        (options ? options.all?{ |k,v| request.original_options[k] == v } : true)
+    end
+
+    # Return canned responses.
+    #
+    # @example Return responses.
+    #   expectation.responses
+    #
+    # @return [ Array<Typhoeus::Response> ] The responses.
+    #
+    # @api private
+    def responses
+      @responses ||= []
+    end
+
+    # Return the response. When there are
+    # multiple responses, they were returned one
+    # by one.
+    #
+    # @example Return response.
+    #   expectation.response
+    #
+    # @return [ Response ] The response.
+    #
+    # @api private
+    def response
+      response = responses.fetch(@response_counter, responses.last)
+      @response_counter += 1
+      response.mock = @from || true
+      response
+    end
+
+    private
+
+    # Check wether the url matches the request url.
+    # The url can be a string, regex or nil. String and
+    # regexp were checked, nil is always true. Else false.
+    #
+    # Nil serves as a placeholder in case you want to match
+    # all urls.
+    def url_match?(request_url)
+      case url
+      when String
+        url == request_url
+      when Regexp
+        !!request_url.match(url)
+      when nil
+        true
+      else
+        false
+      end
+    end
+  end
+end

data/lib/typhoeus/hydra/before.rb

@@ -0,0 +1,30 @@
+module Typhoeus
+  class Hydra
+
+    # This module provides a way to hook into before
+    # a request gets queued in hydra. This is very powerful
+    # and you should be careful because when you accidently
+    # return a falsy value the request won't be executed.
+    #
+    # @api private
+    module Before
+
+      # Overrride queue in order to execute callbacks in
+      # Typhoeus.before. Will break and return when a
+      # callback returns nil or false. Calls super
+      # otherwise.
+      #
+      # @example Queue the request.
+      #   hydra.queue(request)
+      def queue(request)
+        Typhoeus.before.each do |callback|
+          value = callback.call(request)
+          if value.nil? || value == false || value.is_a?(Response)
+            return value
+          end
+        end
+        super
+      end
+    end
+  end
+end

data/lib/typhoeus/hydra/block_connection.rb

@@ -0,0 +1,35 @@
+module Typhoeus
+  class Hydra
+
+    # This module handles the blocked connection request mode on
+    # the hydra side, where only stubbed requests
+    # are allowed.
+    # Connection blocking needs to be turned on:
+    #   Typhoeus.configure do |config|
+    #     config.block_connection = true
+    #   end
+    #
+    # When trying to do real requests a NoStub error
+    # is raised.
+    #
+    # @api private
+    module BlockConnection
+
+      # Overrides queue in order to check before if block connection
+      # is turned on. If thats the case a NoStub error is
+      # raised.
+      #
+      # @example Queue the request.
+      #   hydra.queue(request)
+      #
+      # @param [ Request ] request The request to enqueue.
+      def queue(request)
+        if request.blocked?
+          raise Typhoeus::Errors::NoStub.new(request)
+        else
+          super
+        end
+      end
+    end
+  end
+end

data/lib/typhoeus/hydra/easy_factory.rb

@@ -0,0 +1,79 @@
+module Typhoeus
+  class Hydra
+
+    # This is a Factory for easies to be used in the hydra.
+    # Before an easy is ready to be added to a multi, it needs
+    # to be prepared and the on_complete callback to be set.
+    # This is done by this class.
+    #
+    # @api private
+    class EasyFactory
+
+      # Returns the request provided.
+      #
+      # @return [ Typhoeus::Request ]
+      attr_reader :request
+
+      # Returns the hydra provided.
+      #
+      # @return [ Typhoeus::Hydra ]
+      attr_reader :hydra
+
+      # Create an easy factory.
+      #
+      # @example Create easy factory.
+      #   Typhoeus::Hydra::EasyFactory.new(request, hydra)
+      #
+      # @param [ Request ] request The request to build an easy for.
+      # @param [ Hydra ] hydra The hydra to build an easy for.
+      def initialize(request, hydra)
+        @request = request
+        @hydra = hydra
+      end
+
+      # Return the easy in question.
+      #
+      # @example Return easy.
+      #   easy_factory.easy
+      #
+      # @return [ Ethon::Easy ] The easy.
+      def easy
+        @easy ||= hydra.get_easy
+      end
+
+      # Fabricated and prepared easy.
+      #
+      # @example Prepared easy.
+      #   easy_factory.get
+      #
+      # @return [ Ethon::Easy ] The prepared easy.
+      def get
+        easy.http_request(
+          request.url,
+          request.options.fetch(:method, :get),
+          request.options.reject{|k,_| k==:method}
+        )
+        easy.prepare
+        set_callback
+        easy
+      end
+
+      private
+
+      # Sets on_complete callback on easy in order to be able to
+      # track progress.
+      #
+      # @example Set callback.
+      #   easy_factory.set_callback
+      #
+      # @return [ Ethon::Easy ] The easy.
+      def set_callback
+        easy.on_complete do |easy|
+          request.finish(Response.new(easy.to_hash))
+          hydra.release_easy(easy)
+          hydra.queue(hydra.queued_requests.shift) unless hydra.queued_requests.empty?
+        end
+      end
+    end
+  end
+end

data/lib/typhoeus/hydra/easy_pool.rb

@@ -0,0 +1,42 @@
+module Typhoeus
+  class Hydra
+
+    # The easy pool stores already initialized
+    # easy handles for future use. This is useful
+    # because creating them is quite expensive.
+    #
+    # @api private
+    module EasyPool
+
+      # Return the easy pool.
+      #
+      # @example Return easy pool.
+      #   hydra.easy_pool
+      #
+      # @return [ Array<Ethon::Easy> ] The easy pool.
+      def easy_pool
+        @easy_pool ||= []
+      end
+
+      # Releases easy into pool. The easy handle is
+      # resetted before it gets back in.
+      #
+      # @example Release easy.
+      #   hydra.release_easy(easy)
+      def release_easy(easy)
+        easy.reset
+        easy_pool << easy
+      end
+
+      # Return an easy from pool.
+      #
+      # @example Return easy.
+      #   hydra.get_easy
+      #
+      # @return [ Ethon::Easy ] The easy.
+      def get_easy
+        easy_pool.pop || Ethon::Easy.new
+      end
+    end
+  end
+end

data/lib/typhoeus/hydra/memoizable.rb

@@ -0,0 +1,55 @@
+module Typhoeus
+  class Hydra
+
+    # This module handles the GET request memoization
+    # on the hydra side. Memoization needs to be turned
+    # on:
+    #   Typhoeus.configure do |config|
+    #     config.memoize = true
+    #   end
+    #
+    # @api private
+    module Memoizable
+
+      # Return the memory.
+      #
+      # @example Return the memory.
+      #   hydra.memory
+      #
+      # @return [ Hash ] The memory.
+      def memory
+        @memory ||= {}
+      end
+
+      # Overrides queue in order to check before if request
+      # is memoizable and already in memory. If thats the case,
+      # super is not called, instead the response is set and
+      # the on_complete callback called.
+      #
+      # @example Queue the request.
+      #   hydra.queue(request)
+      #
+      # @param [ Request ] request The request to enqueue.
+      #
+      # @return [ Request ] The queued request.
+      def queue(request)
+        if request.memoizable? && memory.has_key?(request)
+          response = memory[request]
+          request.finish(response, true)
+        else
+          super
+        end
+      end
+
+      # Overrides run to make sure the memory is cleared after
+      # each run.
+      #
+      # @example Run hydra.
+      #   hydra.run
+      def run
+        super
+        memory.clear
+      end
+    end
+  end
+end

data/lib/typhoeus/hydra/queueable.rb

@@ -0,0 +1,48 @@
+module Typhoeus
+  class Hydra
+
+    # This module handles the request queueing on
+    # hydra.
+    #
+    # @api private
+    module Queueable
+
+      # Return the queued requests.
+      #
+      # @example Return queued requests.
+      #  hydra.queued_requests
+      #
+      # @return [ Array<Typhoeus::Request> ] The queued requests.
+      def queued_requests
+        @queued_requests ||= []
+      end
+
+      # Abort the current hydra run as good as
+      # possible. This means that it only
+      # clears the queued requests and can't do
+      # anything about already running requests.
+      #
+      # @example Abort hydra.
+      #  hydra.abort
+      def abort
+        queued_requests.clear
+      end
+
+      # Enqueues a request in order to be performed
+      # by the hydra. This can even be done while
+      # the hydra is running. Also sets hydra on
+      # request.
+      #
+      # @example Queue request.
+      #  hydra.queue(request)
+      def queue(request)
+        request.hydra = self
+        if multi.easy_handles.size < max_concurrency
+          multi.add(Hydra::EasyFactory.new(request, self).get)
+        else
+          queued_requests << request
+        end
+      end
+    end
+  end
+end

data/lib/typhoeus/hydra/runnable.rb

@@ -0,0 +1,21 @@
+module Typhoeus
+  class Hydra
+
+    # This module contains logic to run a hydra.
+    #
+    # @api private
+    module Runnable
+
+      # Start the hydra run.
+      #
+      # @example Start hydra run.
+      #   hydra.run
+      #
+      # @return [ Symbol ] Return value from multi.perform.
+      def run
+        multi.prepare
+        multi.perform
+      end
+    end
+  end
+end

data/lib/typhoeus/hydra/stubbable.rb

@@ -0,0 +1,26 @@
+module Typhoeus
+  class Hydra
+
+    # This module handles stubbing on the hydra side.
+    # It plays well with the block_connection configuration,
+    # which raises when you make a request which is not stubbed.
+    #
+    # @api private
+    module Stubbable
+
+      # Override queue in order to check for matching expecations.
+      # When an expecation is found, super is not called. Instead a
+      # canned response is assigned to the request.
+      #
+      # @example Queue the request.
+      #   hydra.queue(request)
+      def queue(request)
+        if expectation = Expectation.find_by(request)
+          request.finish(expectation.response)
+        else
+          super
+        end
+      end
+    end
+  end
+end