typhoeus 0.5.0.pre → 0.5.0.rc

data/CHANGELOG.md

@@ -20,6 +20,7 @@ Major Changes:
     for a description.
   * The following classes were deleted because they do not seemed to be uesed at all. If that
     turns out to be wrong, they will be restored: `Typhoeus::Filter`, `Typhoeus::Remote`, `Typhoeus::RemoteMethod`, `Typhoeus::RemoteProxyObject`
+  * `Typhoeus::Easy` and `Typhoeus::Multi` are now `Ethon::Easy` and `Ethon::Multi`
 
 * Request shortcuts: `Typhoeus.get("www.google.de")`
 * Global configuration:
@@ -32,6 +33,17 @@ end
 * No more Response#headers_hash, instead response#header returning the last
   header and response#redirections returning the responses with headers
   generated through redirections
+* Instead of defining the same callbacks on every request, you can define global callbacks:
+
+```ruby
+Typhoeus.on_complete { p "yay" }
+```
+
+* The stubbing interface changed slightly. You now have the same syntax as for requests:
+
+```ruby
+Typhoeus.stub(url, options).and_return(response)
+```
 
 Enhancements:
 

data/Gemfile

@@ -1,3 +1,19 @@
 source :rubygems
-
 gemspec
+
+gem "rake"
+
+group :development, :test do
+  gem "rspec", "~> 2.11"
+
+  gem "sinatra", "~> 1.3"
+  gem "json"
+
+  if RUBY_PLATFORM == "java"
+    gem "spoon"
+  end
+
+  unless ENV["CI"]
+    gem "guard-rspec", "~> 0.7"
+  end
+end

data/README.md

@@ -20,8 +20,8 @@ hydra.run
 
 ## Project Tracking
 
-* [Documentation](http://rubydoc.info/github/typhoeus/typhoeus)
-* [Website](http://typhoeus.github.com/)
+* [Documentation](http://rubydoc.info/github/typhoeus/typhoeus) (github master)
+* [Website](http://typhoeus.github.com/) (v0.4.2)
 * [Mailinglist](http://groups.google.com/group/typhoeus)
 
 ## LICENSE

data/lib/typhoeus.rb

@@ -2,41 +2,103 @@ require 'digest/sha2'
 require 'ethon'
 
 require 'typhoeus/config'
+require 'typhoeus/errors'
+require 'typhoeus/expectation'
+require 'typhoeus/hydra'
 require 'typhoeus/request'
 require 'typhoeus/response'
-require 'typhoeus/hydra'
 require 'typhoeus/version'
 
 # Typhoeus is a http client library based on Ethon which
 # wraps libcurl.
 #
-# If you want to make a single request, go with:
-#   Typhoeus.get("www.example.com")
+# @example (see Typhoeus::Request)
+# @example (see Typhoeus::Hydra)
 #
-# When you looking for firing a bunch of requests automatically
-# choose the hydra:
+# @see Typhoeus::Request
+# @see Typhoeus::Hydra
 #
-#   hydra = Typhoeus::Hydra.new
-#   requests = (0..9).map{ Typhoeus::Request.new("www.example.com") }
-#   requests.each{ |request| hydra.queue(request) }
-#   hydra.run
+# @since 0.5.0
 module Typhoeus
   extend self
-  extend Hydras::EasyPool
-  extend Requests::Actions
+  extend Hydra::EasyPool
+  extend Request::Actions
+  extend Request::Callbacks::Types
 
   # The default typhoeus user agent.
   USER_AGENT = "Typhoeus - https://github.com/typhoeus/typhoeus"
 
   # Set the Typhoeus configuration options by passing a block.
   #
-  # @example Set the configuration options.
-  #   Typhoeus.configure do |config|
-  #     config.verbose = true
-  #   end
+  # @example (see Typhoeus::Config)
+  #
+  # @yield [ Typhoeus::Config ]
   #
-  # @return [ Config ] The configuration object.
+  # @return [ Typhoeus::Config ] The configuration.
+  #
+  # @see Typhoeus::Config
   def configure
     yield Config
   end
+
+  # Stub out specific request.
+  #
+  # @example (see Typhoeus::Expectation)
+  #
+  # @param [ String ] url The url to stub out.
+  # @param [ Hash ] options The options to stub out.
+  #
+  # @return [ Typhoeus::Expectation ] The expecatation.
+  #
+  # @see Typhoeus::Expectation
+  def stub(url, options = {})
+    expectation = Expectation.all.find{ |e| e.url == url && e.options == options }
+    return expectation if expectation
+
+    Expectation.new(url, options).tap do |new_expectation|
+      Expectation.all << new_expectation
+    end
+  end
+
+  # Add before callbacks.
+  #
+  # @example Add before callback.
+  #   Typhoeus.before { |request| p request.url }
+  #
+  # @param [ Block ] block The callback.
+  #
+  # @yield [ Typhoeus::Request ]
+  #
+  # @return [ Array<Block> ] All before blocks.
+  def before(&block)
+    @before ||= []
+    @before << block if block_given?
+    @before
+  end
+
+  # Execute given block as if block connection is turned off.
+  # The old block connection state is restored afterwards.
+  #
+  # @example Make a real request, no matter if its blocked.
+  #   Typhoeus::Config.block_connection = true
+  #   Typhoeus.get("www.example.com").code
+  #   #=> raise Typhoeus::Errors::NoStub
+  #
+  #   Typhoeus.with_connection do
+  #     Typhoeus.get("www.example.com").code
+  #     #=> :ok
+  #   end
+  #
+  # @param [ Block ] block The block to execute.
+  #
+  # @return [ Object ] Returns the return value of block.
+  #
+  # @see Typhoeus::Config#block_connection
+  def with_connection
+    old = Config.block_connection
+    Config.block_connection = false
+    result = yield if block_given?
+    Config.block_connection = old
+    result
+  end
 end

data/lib/typhoeus/config.rb

@@ -1,11 +1,47 @@
 module Typhoeus
 
   # The Typhoeus configuration used to set global
-  # options. Available options:
-  # * verbose
-  # * memoize.
+  # options.
+  # @example Set the configuration options within a block.
+  #   Typhoeus.configure do |config|
+  #     config.verbose = true
+  #   end
+  #
+  # @example Set the configuration direct.
+  #   Typhoeus::Config.verbose = true
   module Config
     extend self
-    attr_accessor :verbose, :memoize
+
+    # Defines wether the connection is blocked.
+    # Defaults to false. When set to true, only
+    # stubbed requests are allowed. A
+    # {Typhoeus::Errors::NoStub} error is raised,
+    # when trying to do a real request. Its possible
+    # to work around inside
+    # {Typhoeus#with_connection}.
+    #
+    # @return [ Boolean ]
+    #
+    # @see Typhoeus::Request::BlockConnection
+    # @see Typhoeus::Hydra::BlockConnection
+    # @see Typhoeus#with_connection
+    # @see Typhoeus::Errors::NoStub
+    attr_accessor :block_connection
+
+    # Defines wether GET requests are memoized when using the {Typhoeus::Hydra}.
+    #
+    # @return [ Boolean ]
+    #
+    # @see Typhoeus::Hydra
+    # @see Typhoeus::Hydra::Memoizable
+    attr_accessor :memoize
+
+    # Defines wether curls debug output is shown.
+    # Unfortunately it prints to stderr.
+    #
+    # @return [ Boolean ]
+    #
+    # @see http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTVERBOSE
+    attr_accessor :verbose
   end
 end

data/lib/typhoeus/errors.rb

@@ -0,0 +1,9 @@
+require 'typhoeus/errors/typhoeus_error'
+require 'typhoeus/errors/no_stub'
+
+module Typhoeus
+
+  # This namespace contains all errors raised by typhoeus.
+  module Errors
+  end
+end

data/lib/typhoeus/errors/no_stub.rb

@@ -0,0 +1,12 @@
+module Typhoeus
+  module Errors
+
+    # Raises when block connection is turned on
+    # and making a real request.
+    class NoStub < TyphoeusError
+      def initialize(request)
+        super("The connection is blocked and no stub defined.")
+      end
+    end
+  end
+end

data/lib/typhoeus/errors/typhoeus_error.rb

@@ -0,0 +1,8 @@
+module Typhoeus
+  module Errors
+
+    # Default typhoeus error class for all custom errors.
+    class TyphoeusError < StandardError
+    end
+  end
+end

data/lib/typhoeus/expectation.rb

@@ -0,0 +1,174 @@
+module Typhoeus
+
+  # This class represents an expectation. It is part
+  # of the stubbing mechanism. An expectation contains
+  # an url and options, like a request. They were compared
+  # to the request url and options in order to evaluate
+  # wether they match. If thats the case, the attached
+  # responses were returned one by one.
+  #
+  # @example Stub a request and get specified response.
+  #   expected = Typhoeus::Response.new
+  #   Typhoeus.stub("www.example.com").and_return(expected)
+  #
+  #   actual = Typhoeus.get("www.example.com")
+  #   expected == actual
+  #   #=> true
+  class Expectation
+
+    # @api private
+    attr_reader :url
+
+    # @api private
+    attr_reader :options
+
+    # @api private
+    attr_reader :from
+
+    class << self
+
+      # Returns all expectations.
+      #
+      # @example Return expectations.
+      #   Typhoeus::Expectation.all
+      #
+      # @return [ Array<Typhoeus::Expectation> ] The expectations.
+      def all
+        @expectations ||= []
+      end
+
+      # Clears expectations. This is handy while
+      # testing and you want to make sure, that
+      # you don't get canned responses.
+      #
+      # @example Clear expectations.
+      #   Typhoeus::Expectation.clear
+      def clear
+        all.clear
+      end
+
+      # Returns expecation matching the provided
+      # request.
+      #
+      # @example Find expectation.
+      #   Typhoeus::Expectation.find_by(request)
+      #
+      # @return [ Expectation ] The matching expectation.
+      #
+      # @api private
+      def find_by(request)
+        all.find do |expectation|
+          expectation.matches?(request)
+        end
+      end
+    end
+
+    # Creates an expactation.
+    #
+    # @example Create expactation.
+    #   Typhoeus::Expectation.new(url)
+    #
+    # @return [ Expectation ] The created expactation.
+    #
+    # @api private
+    def initialize(url, options = {})
+      @url = url
+      @options = options
+      @response_counter = 0
+      @from = nil
+    end
+
+    # Set from value to mark an expecation. Useful for
+    # other libraries, eg. webmock.
+    #
+    # @example Mark expecation.
+    #   expecation.from(:webmock)
+    #
+    # @param [ String ] value Value to set.
+    #
+    # @return [ Expectation ] Returns self.
+    #
+    # @api private
+    def stubbed_from(value)
+      @from = value
+      self
+    end
+
+    # Specify what should be returned,
+    # when this expactation is hit.
+    #
+    # @example Add response.
+    #   expectation.and_return(response)
+    #
+    # @return [ void ]
+    def and_return(response)
+      responses << response
+    end
+
+    # Checks wether this expectation matches
+    # the provided request.
+    #
+    # @example Check if request matches.
+    #   expectation.matches? request
+    #
+    # @param [ Request ] request The request to check.
+    #
+    # @return [ Boolean ] True when matches, else false.
+    #
+    # @api private
+    def matches?(request)
+      url_match?(request.url) &&
+        (options ? options.all?{ |k,v| request.original_options[k] == v } : true)
+    end
+
+    # Return canned responses.
+    #
+    # @example Return responses.
+    #   expectation.responses
+    #
+    # @return [ Array<Typhoeus::Response> ] The responses.
+    #
+    # @api private
+    def responses
+      @responses ||= []
+    end
+
+    # Return the response. When there are
+    # multiple responses, they were returned one
+    # by one.
+    #
+    # @example Return response.
+    #   expectation.response
+    #
+    # @return [ Response ] The response.
+    #
+    # @api private
+    def response
+      response = responses.fetch(@response_counter, responses.last)
+      @response_counter += 1
+      response.mock = @from || true
+      response
+    end
+
+    private
+
+    # Check wether the url matches the request url.
+    # The url can be a string, regex or nil. String and
+    # regexp were checked, nil is always true. Else false.
+    #
+    # Nil serves as a placeholder in case you want to match
+    # all urls.
+    def url_match?(request_url)
+      case url
+      when String
+        url == request_url
+      when Regexp
+        !!request_url.match(url)
+      when nil
+        true
+      else
+        false
+      end
+    end
+  end
+end

data/lib/typhoeus/hydra.rb

@@ -1,37 +1,94 @@
-require 'typhoeus/hydras/easy_factory'
-require 'typhoeus/hydras/easy_pool'
-require 'typhoeus/hydras/memoizable'
-require 'typhoeus/hydras/queueable'
-require 'typhoeus/hydras/runnable'
+require 'typhoeus/hydra/before'
+require 'typhoeus/hydra/block_connection'
+require 'typhoeus/hydra/easy_factory'
+require 'typhoeus/hydra/easy_pool'
+require 'typhoeus/hydra/memoizable'
+require 'typhoeus/hydra/queueable'
+require 'typhoeus/hydra/runnable'
+require 'typhoeus/hydra/stubbable'
 
 module Typhoeus
 
   # Hydra manages making parallel HTTP requests. This
-  # is archived by using libcurls multi interface. The
-  # benefits are that you don't have to worry running
+  # is achieved by using libcurls multi interface:
+  # http://curl.haxx.se/libcurl/c/libcurl-multi.html
+  # The benefits are that you don't have to worry running
   # the requests by yourself.
+  #
+  # Hydra will also handle how many requests you can
+  # make in parallel. Things will get flakey if you
+  # try to make too many requests at the same time.
+  # The built in limit is 200. When more requests than
+  # that are queued up, hydra will save them for later
+  # and start the requests as others are finished. You
+  # can raise or lower the concurrency limit through
+  # the Hydra constructor.
+  #
+  # Regarding the asynchronous behavior of the hydra,
+  # it is important to know that this is completely hidden
+  # from the developer and you are free to apply
+  # whatever technique you want to your code. That should not
+  # conflict with libcurls internal concurrency mechanism.
+  #
+  # @example Use the hydra to do multiple requests.
+  #   hydra = Typhoeus::Hydra.new
+  #   requests = (0..9).map{ Typhoeus::Request.new("www.example.com") }
+  #   requests.each{ |request| hydra.queue(request) }
+  #   hydra.run
   class Hydra
-    include Hydras::EasyPool
-    include Hydras::Queueable
-    include Hydras::Runnable
-    include Hydras::Memoizable
+    include Hydra::EasyPool
+    include Hydra::Queueable
+    include Hydra::Runnable
+    include Hydra::Memoizable
+    include Hydra::BlockConnection
+    include Hydra::Stubbable
+    include Hydra::Before
 
-    attr_reader :max_concurrency, :multi
+    # @example Set max_concurrency.
+    #   Typhoeus::Hydra.new(:max_concurrency => 20)
+    attr_reader :max_concurrency
 
-    # Create a new hydra.
+    # @api private
+    attr_reader :multi
+
+    class << self
+
+      # Returns a memoized hydra instance.
+      #
+      # @example Get a hydra.
+      #   Typhoeus::Hydra.hydra
+      #
+      # @return [Typhoeus::Hydra] A new hydra.
+      #
+      # @deprecated This is only for convenience because so
+      #   much external code relies on it.
+      def hydra
+        @hydra ||= new
+      end
+    end
+
+    # Create a new hydra. All
+    # {http://rubydoc.info/github/typhoeus/ethon/Ethon/Multi#initialize-instance_method Ethon::Multi#initialize}
+    # options are also available.
     #
     # @example Create a hydra.
     #   Typhoeus::Hydra.new
     #
+    # @example Create a hydra with max_concurrency.
+    #   Typhoeus::Hydra.new(:max_concurrency => 20)
+    #
     # @param [ Hash ] options The options hash.
     #
     # @option options :max_concurrency [ Integer ] Number
     #  of max concurrent connections to create. Default is
     #  200.
+    #
+    # @see http://rubydoc.info/github/typhoeus/ethon/Ethon/Multi#initialize-instance_method
+    #   Ethon::Multi#initialize
     def initialize(options = {})
       @options = options
       @max_concurrency = @options.fetch(:max_concurrency, 200)
-      @multi = Ethon::Multi.new
+      @multi = Ethon::Multi.new(options.reject{|k,_| k==:max_concurrency})
     end
   end
 end