typhoeus 0.4.2 → 1.4.0

data/Rakefile

@@ -1,13 +1,18 @@
-$LOAD_PATH.unshift(File.dirname(__FILE__))
+require "bundler"
+Bundler.setup
 
-require 'rspec/core/rake_task'
-RSpec::Core::RakeTask.new do |t|
+require "rake"
+require "rspec/core/rake_task"
+$LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
+require "typhoeus/version"
+
+task :gem => :build
+task :build do
+  system "gem build typhoeus.gemspec"
 end
 
-task :install do
-  rm_rf "*.gem"
-  puts `gem build typhoeus.gemspec`
-  puts `gem install typhoeus-*.gem`
+task :install => :build do
+  system "gem install typhoeus-#{Typhoeus::VERSION}.gem"
 end
 
 task :release => :build do
@@ -16,14 +21,18 @@ task :release => :build do
   system "gem push typhoeus-#{Typhoeus::VERSION}.gem"
 end
 
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.verbose = false
+  t.ruby_opts = "-W -I./spec -rspec_helper"
+end
+
 desc "Start up the test servers"
-task :start_test_servers do
-  require 'spec/support/typhoeus_localhost_server'
+task :start do
+  require_relative 'spec/support/boot'
   begin
-    TyphoeusLocalhostServer.start_servers!(:rake)
+    Boot.start_servers(:rake)
   rescue Exception
   end
 end
 
-desc "Build Typhoeus and run all the tests."
-task :default => [:spec]
+task :default => :spec

data/UPGRADE.md

@@ -0,0 +1,55 @@
+# Upgrade guide
+
+## 0.5
+
+### Options
+
+Fix the option names, because some were renamed. The errors should point you in the right direction:
+
+```ruby
+Typhoeus.get("www.example.com", follow_location: true)
+# Ethon::Errors::InvalidOption: The option: follow_location is invalid.
+# Please try followlocation instead of follow_location.
+# ... [Backtrace]
+
+Typhoeus.get("www.example.com", followlocation: true).code
+#=> 200
+```
+
+### Headers
+
+`Response#headers` returns a hash now and replaces `Response#headers_hash`, use `Response#response_headers` for the raw string:
+
+```ruby
+Typhoeus.get("www.example.com", followlocation: true).headers
+#=> {
+#      "date"=>"Tue, 06 Nov 2012 09:07:27 GMT",
+#      "server"=>"Apache/2.2.3 (CentOS)",
+#      "last-modified"=>"Wed, 09 Feb 2011 17:13:15 GMT",
+#      "vary"=>"Accept-Encoding",
+#      "connection"=>"close",
+#      "content-type"=>"text/html; charset=UTF-8"
+#   }
+
+Typhoeus.get("www.example.com", followlocation: true).response_headers
+#=> "HTTP/1.0 302 Found\r\nLocation: http://www.iana.org/domains/example/ [...]"
+```
+
+### Params vs body
+
+Make sure every request sends proper params and body (especially POST/PUT). `:params` becomes url parameter and `:body` request body. Before params for POST was smashed into the body.
+
+### Configuration
+
+Create a global configuration in case you want to turn on verbose, memoize or block_connection:
+
+```ruby
+Typhoeus.configure do |config|
+  config.verbose = true
+  config.memoize = true
+end
+```
+
+### Docs
+
+When in doubt, read the [docs](http://rubydoc.info/github/typhoeus/typhoeus/frames/Typhoeus) or the [code](https://www.github.com/typhoeus).

data/lib/rack/typhoeus/middleware/params_decoder/helper.rb

@@ -0,0 +1,76 @@
+module Rack
+  module Typhoeus
+    module Middleware
+      class ParamsDecoder
+        module Helper
+
+          # Recursively decodes Typhoeus encoded arrays in given Hash.
+          #
+          # @example Use directly in a Rails controller.
+          #    class ApplicationController
+          #       before_filter :decode_typhoeus_arrays
+          #    end
+          #
+          # @author Dwayne Macgowan
+          #
+          def decode_typhoeus_arrays
+            decode!(params)
+          end
+
+          # Recursively decodes Typhoeus encoded arrays in given Hash.
+          #
+          # @param hash [Hash]. This Hash will be modified!
+          #
+          # @return [Hash] Hash with properly decoded nested arrays.
+          def decode!(hash)
+            return hash unless hash.is_a?(Hash)
+            hash.each_pair do |key,value|
+              if value.is_a?(Hash)
+                decode!(value)
+                hash[key] = convert(value)
+              end
+            end
+            hash
+          end
+
+          def decode(hash)
+            decode!(hash.dup)
+          end
+
+          private
+
+          # Checks if Hash is an Array encoded as a Hash.
+          # Specifically will check for the Hash to have this
+          # form: {'0' => v0, '1' => v1, .., 'n' => vN }
+          #
+          # @param hash [Hash]
+          #
+          # @return [Boolean] True if its a encoded Array, else false.
+          def encoded?(hash)
+            return false if hash.empty?
+	          if hash.keys.size > 1
+              keys = hash.keys.map{|i| i.to_i if i.respond_to?(:to_i)}.sort
+              keys == hash.keys.size.times.to_a
+	          else
+              hash.keys.first =~ /0/
+	          end
+          end
+
+          # If the Hash is an array encoded by typhoeus an array is returned
+          # else the self is returned
+          #
+          # @param hash [Hash] The Hash to convert into an Array.
+          #
+          # @return [Arraya/Hash]
+          def convert(hash)
+            if encoded?(hash)
+              hash.sort{ |a, b| a[0].to_i <=> b[0].to_i }.map{ |key, value| value }
+            else
+              hash
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rack/typhoeus/middleware/params_decoder.rb

@@ -0,0 +1,57 @@
+require 'rack/typhoeus/middleware/params_decoder/helper'
+
+module Rack
+  module Typhoeus
+    module Middleware
+
+      # This Rack middleware takes care of the proper deserialization of
+      # the nested params encoded by Typhoeus.
+      #
+      # @example Require the railtie when using Rails.
+      #   require 'typhoeus/railtie'
+      #
+      # @example Include the middleware for Rack based applications.
+      #   use Rack::Typhoeus::Middleware::ParamsDecoder
+      #
+      # @example Use the helper directly. Not recommended as b/c the interface might change.
+      #   require 'rack/typhoeus/middleware/params_decoder/helper'
+      #   include Rack::Typhoeus::Middleware::ParamsDecoder::Helper
+      #   decode!(params)
+      #
+      # @author Dwayne Macgowan
+      # @since 0.5.4
+      class ParamsDecoder
+        include ParamsDecoder::Helper
+
+        def initialize(app)
+          @app = app
+        end
+
+        def call(env)
+          req = Rack::Request.new(env)
+          decode(req.params).each_pair { |k, v| update_params req, k, v }
+          @app.call(env)
+        end
+
+        private
+
+        # Persist params change in environment. Extracted from:
+        # https://github.com/rack/rack/blob/master/lib/rack/request.rb#L243
+        def update_params(req, k, v)
+          found = false
+          if req.GET.has_key?(k)
+            found = true
+            req.GET[k] = v
+          end
+          if req.POST.has_key?(k)
+            found = true
+            req.POST[k] = v
+          end
+          unless found
+            req.GET[k] = v
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rack/typhoeus.rb

@@ -0,0 +1 @@
+require "rack/typhoeus/middleware/params_decoder"

data/lib/typhoeus/adapters/faraday.rb

@@ -0,0 +1,180 @@
+require 'faraday'
+
+module Faraday # :nodoc:
+  class Adapter # :nodoc:
+
+    # Adapter to use Faraday with Typhoeus.
+    #
+    # @example Use Typhoeus.
+    #   require 'faraday'
+    #   require 'typhoeus'
+    #   require 'typhoeus/adapters/faraday'
+    #
+    #   conn = Faraday.new(url: "www.example.com") do |faraday|
+    #     faraday.adapter :typhoeus
+    #
+    #     # You can include Typhoeus options to be used for every request
+    #     # faraday.adapter :typhoeus, forbid_reuse: true, maxredirs: 1
+    #   end
+    #
+    #   response = conn.get("/")
+    class Typhoeus < Faraday::Adapter
+      self.supports_parallel = true
+
+      (class << self; self; end).instance_eval do
+        remove_method :setup_parallel_manager if method_defined? :setup_parallel_manager
+      end
+
+      remove_method :call              if method_defined? :call
+      remove_method :perform_request   if method_defined? :perform_request
+      remove_method :request           if method_defined? :request
+      remove_method :read_body         if method_defined? :read_body
+      remove_method :configure_ssl     if method_defined? :configure_ssl
+      remove_method :configure_proxy   if method_defined? :configure_proxy
+      remove_method :configure_timeout if method_defined? :configure_timeout
+      remove_method :configure_socket  if method_defined? :configure_socket
+      remove_method :parallel?         if method_defined? :parallel?
+
+      # Initialize the Typhoeus adapter
+      #
+      # @param [ App ] app Farday app
+      # @option [ Hash ] adapter_options Typhoeus options
+      #
+      # @return [ void ]
+      def initialize(app, adapter_options = {})
+        super(app)
+        @adapter_options = adapter_options
+      end
+
+      # Setup Hydra with provided options.
+      #
+      # @example Setup Hydra.
+      #   Faraday::Adapter::Typhoeus.setup_parallel_manager
+      #   #=> #<Typhoeus::Hydra ... >
+      #
+      # @param (see Typhoeus::Hydra#initialize)
+      # @option (see Typhoeus::Hydra#initialize)
+      #
+      # @return [ Typhoeus::Hydra ] The hydra.
+      def self.setup_parallel_manager(options = {})
+        ::Typhoeus::Hydra.new(options)
+      end
+
+      dependency 'typhoeus'
+
+      # Hook into Faraday and perform the request with Typhoeus.
+      #
+      # @param [ Hash ] env The environment.
+      #
+      # @return [ void ]
+      def call(env)
+        super
+        perform_request env
+        @app.call env
+      end
+
+      private
+
+      def perform_request(env)
+        if parallel?(env)
+          env[:parallel_manager].queue request(env)
+        else
+          request(env).run
+        end
+      end
+
+      def request(env)
+        read_body env
+
+        req = typhoeus_request(env)
+
+        configure_ssl     req, env
+        configure_proxy   req, env
+        configure_timeout req, env
+        configure_socket  req, env
+
+        req.on_complete do |resp|
+          if resp.timed_out?
+            env[:typhoeus_timed_out] = true
+            unless parallel?(env)
+              raise Faraday::TimeoutError, "request timed out"
+            end
+          elsif (resp.response_code == 0) || ((resp.return_code != :ok) && !resp.mock?)
+            env[:typhoeus_connection_failed] = true
+            env[:typhoeus_return_message] = resp.return_message
+            unless parallel?(env)
+              raise Faraday::ConnectionFailed, resp.return_message
+            end
+          end
+
+          save_response(env, resp.code, resp.body) do |response_headers|
+            response_headers.parse resp.response_headers
+          end
+          # in async mode, :response is initialized at this point
+          env[:response].finish(env) if parallel?(env)
+        end
+
+        req
+      end
+
+      def typhoeus_request(env)
+        opts = {
+          :method => env[:method],
+          :body => env[:body],
+          :headers => env[:request_headers]
+        }.merge(@adapter_options)
+
+        ::Typhoeus::Request.new(env[:url].to_s, opts)
+      end
+
+      def read_body(env)
+        env[:body] = env[:body].read if env[:body].respond_to? :read
+      end
+
+      def configure_ssl(req, env)
+        ssl = env[:ssl]
+
+        verify_p = (ssl && ssl.fetch(:verify, true))
+
+        ssl_verifyhost = verify_p ? 2 : 0
+        req.options[:ssl_verifyhost] = ssl_verifyhost
+        req.options[:ssl_verifypeer] = verify_p
+        req.options[:sslversion] = ssl[:version]     if ssl[:version]
+        req.options[:sslcert]    = ssl[:client_cert] if ssl[:client_cert]
+        req.options[:sslkey]     = ssl[:client_key]  if ssl[:client_key]
+        req.options[:cainfo]     = ssl[:ca_file]     if ssl[:ca_file]
+        req.options[:capath]     = ssl[:ca_path]     if ssl[:ca_path]
+        client_cert_passwd_key   = [:client_cert_passwd, :client_certificate_password].detect { |name| ssl.key?(name) }
+        req.options[:keypasswd]  = ssl[client_cert_passwd_key] if client_cert_passwd_key
+      end
+
+      def configure_proxy(req, env)
+        proxy = env[:request][:proxy]
+        return unless proxy
+
+        req.options[:proxy] = "#{proxy[:uri].scheme}://#{proxy[:uri].host}:#{proxy[:uri].port}"
+
+        if proxy[:user] && proxy[:password]
+          req.options[:proxyauth] = :any
+          req.options[:proxyuserpwd] = "#{proxy[:user]}:#{proxy[:password]}"
+        end
+      end
+
+      def configure_timeout(req, env)
+        env_req = env[:request]
+        req.options[:timeout_ms] = (env_req[:timeout] * 1000).to_i             if env_req[:timeout]
+        req.options[:connecttimeout_ms] = (env_req[:open_timeout] * 1000).to_i if env_req[:open_timeout]
+      end
+
+      def configure_socket(req, env)
+        if bind = env[:request][:bind]
+          req.options[:interface] = bind[:host]
+        end
+      end
+
+      def parallel?(env)
+        !!env[:parallel_manager]
+      end
+    end
+  end
+end

data/lib/typhoeus/cache/dalli.rb

@@ -0,0 +1,28 @@
+module Typhoeus
+  module Cache
+    # This module provides a simple way to cache HTTP responses using Dalli.
+    class Dalli
+      # @example Set Dalli as the Typhoeus cache backend
+      #   Typhoeus::Config.cache = Typhoeus::Cache::Dalli.new
+      #
+      # @param [ Dalli::Client ] client
+      #   A connection to the cache server. Defaults to `Dalli::Client.new`
+      # @param [ Hash ] options
+      #   Options
+      # @option options [ Integer ] :default_ttl
+      #   The default TTL of cached responses in seconds, for requests which do not set a cache_ttl.
+      def initialize(client = ::Dalli::Client.new, options = {})
+        @client = client
+        @default_ttl = options[:default_ttl]
+      end
+
+      def get(request)
+        @client.get(request.cache_key)
+      end
+
+      def set(request, response)
+        @client.set(request.cache_key, response, request.cache_ttl || @default_ttl)
+      end
+    end
+  end
+end

data/lib/typhoeus/cache/rails.rb

@@ -0,0 +1,28 @@
+module Typhoeus
+  module Cache
+    # This module provides a simple way to cache HTTP responses in using the Rails cache.
+    class Rails
+      # @example Use the Rails cache setup to cache Typhoeus responses.
+      #   Typhoeus::Config.cache = Typhoeus::Cache::Rails.new
+      #
+      # @param [ ActiveSupport::Cache::Store ] cache
+      #   A Rails cache backend. Defaults to Rails.cache.
+      # @param [ Hash ] options
+      #   Options
+      # @option options [ Integer ] :default_ttl
+      #   The default TTL of cached responses in seconds, for requests which do not set a cache_ttl.
+      def initialize(cache = ::Rails.cache, options = {})
+        @cache = cache
+        @default_ttl = options[:default_ttl]
+      end
+
+      def get(request)
+        @cache.read(request)
+      end
+
+      def set(request, response)
+        @cache.write(request.cache_key, response, :expires_in => request.cache_ttl || @default_ttl)
+      end
+    end
+  end
+end

data/lib/typhoeus/cache/redis.rb

@@ -0,0 +1,35 @@
+module Typhoeus
+  module Cache
+    # This module provides a simple way to cache HTTP responses in Redis.
+    class Redis
+      # @example Set Redis as the Typhoeus cache backend
+      #   Typhoeus::Config.cache = Typhoeus::Cache::Redis.new
+      #
+      # @param [ Redis ] redis
+      #   A connection to Redis. Defaults to `Redis.new`, which uses the
+      #   `REDIS_URL` environment variable to connect
+      # @param [ Hash ] options
+      #   Options
+      # @option options [ Integer ] :default_ttl
+      #   The default TTL of cached responses in seconds, for requests which do not set a cache_ttl.
+      def initialize(redis = ::Redis.new, options = {})
+        @redis = redis
+        @default_ttl = options[:default_ttl]
+      end
+
+      def get(request)
+        serialized_response = @redis.get(request.cache_key)
+        return unless serialized_response
+        Marshal.load(serialized_response)
+      end
+
+      def set(request, response)
+        ttl = request.cache_ttl || @default_ttl
+        key = request.cache_key
+        serialized_response = Marshal.dump(response)
+        @redis.set(key, serialized_response)
+        @redis.expire(key, ttl) if ttl
+      end
+    end
+  end
+end

data/lib/typhoeus/config.rb

@@ -0,0 +1,69 @@
+module Typhoeus
+
+  # The Typhoeus configuration used to set global
+  # options.
+  # @example Set the configuration options within a block.
+  #   Typhoeus.configure do |config|
+  #     config.verbose = true
+  #   end
+  #
+  # @example Set the configuration directly.
+  #   Typhoeus::Config.verbose = true
+  module Config
+    extend self
+
+    # Defines whether the connection is blocked.
+    # Defaults to false. When set to true, only
+    # stubbed requests are allowed. A
+    # {Typhoeus::Errors::NoStub} error is raised,
+    # when trying to do a real request. It's possible
+    # to work around inside
+    # {Typhoeus.with_connection}.
+    #
+    # @return [ Boolean ]
+    #
+    # @see Typhoeus::Request::BlockConnection
+    # @see Typhoeus::Hydra::BlockConnection
+    # @see Typhoeus#with_connection
+    # @see Typhoeus::Errors::NoStub
+    attr_accessor :block_connection
+
+    # Defines whether GET requests are memoized when using the {Typhoeus::Hydra}.
+    #
+    # @return [ Boolean ]
+    #
+    # @see Typhoeus::Hydra
+    # @see Typhoeus::Hydra::Memoizable
+    attr_accessor :memoize
+
+    # Defines whether curls debug output is shown.
+    # Unfortunately it prints to stderr.
+    #
+    # @return [ Boolean ]
+    #
+    # @see http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTVERBOSE
+    attr_accessor :verbose
+
+    # Defines whether requests are cached.
+    #
+    # @return [ Object ]
+    #
+    # @see Typhoeus::Hydra::Cacheable
+    # @see Typhoeus::Request::Cacheable
+    attr_accessor :cache
+
+    # Defines whether to use a default user agent.
+    #
+    # @return [ String ]
+    #
+    # @see Typhoeus::Request#set_defaults
+    attr_accessor :user_agent
+
+    # Defines wether to use a proxy server for every request.
+    #
+    # @return [ String ]
+    #
+    # @see Typhoeus::Request#set_defaults
+    attr_accessor :proxy
+  end
+end