dcu-typhoeus 0.4.0

data/lib/typhoeus/request.rb

@@ -0,0 +1,278 @@
+require 'uri'
+
+module Typhoeus
+  class Request
+    ACCESSOR_OPTIONS = [
+      :method,
+      :params,
+      :body,
+      :headers,
+      :cache_key_basis,
+      :connect_timeout,
+      :timeout,
+      :user_agent,
+      :response,
+      :cache_timeout,
+      :follow_location,
+      :max_redirects,
+      :proxy,
+      :proxy_username,
+      :proxy_password,
+      :disable_ssl_peer_verification,
+      :disable_ssl_host_verification,
+      :interface,
+      :ssl_cert,
+      :ssl_cert_type,
+      :ssl_key,
+      :ssl_key_type,
+      :ssl_key_password,
+      :ssl_cacert,
+      :ssl_capath,
+      :ssl_version,
+      :verbose,
+      :username,
+      :password,
+      :auth_method,
+      :user_agent,
+      :proxy_auth_method,
+      :proxy_type
+    ]
+
+    attr_accessor *ACCESSOR_OPTIONS
+
+    # Initialize a new Request
+    #
+    # Options:
+    # * +url+ : Endpoint (URL) of the request
+    # * +options+   : A hash containing options among :
+    # ** +:method+  : :get (default) / :post / :put
+    # ** +:params+  : params as a Hash
+    # ** +:body+
+    # ** +:timeout+ : timeout (ms)
+    # ** +:interface+ : interface or ip address (string)
+    # ** +:connect_timeout+ : connect timeout (ms)
+    # ** +:headers+  : headers as Hash
+    # ** +:cache_timeout+ : cache timeout (ms)
+    # ** +:follow_location
+    # ** +:max_redirects
+    # ** +:proxy
+    # ** +:disable_ssl_peer_verification
+    # ** +:disable_ssl_host_verification
+    # ** +:ssl_cert
+    # ** +:ssl_cert_type
+    # ** +:ssl_key
+    # ** +:ssl_key_type
+    # ** +:ssl_key_password
+    # ** +:ssl_cacert
+    # ** +:ssl_capath
+    # ** +:verbose
+    # ** +:username
+    # ** +:password
+    # ** +:auth_method
+    #
+    def initialize(url, options = {})
+      @url = url
+      @method           = options[:method] || :get
+      @params           = options[:params]
+      @body             = options[:body]
+      @timeout          = safe_to_i(options[:timeout])
+      @connect_timeout  = safe_to_i(options[:connect_timeout])
+      @interface        = options[:interface]
+      @headers          = options[:headers] || {}
+
+      if options.key?(:user_agent)
+        @headers['User-Agent'] = options[:user_agent]
+      end
+
+      @cache_timeout    = safe_to_i(options[:cache_timeout])
+      @follow_location  = options[:follow_location]
+      @max_redirects    = options[:max_redirects]
+      @proxy            = options[:proxy]
+      @proxy_type       = options[:proxy_type]
+      @proxy_username   = options[:proxy_username]
+      @proxy_password   = options[:proxy_password]
+      @proxy_auth_method = options[:proxy_auth_method]
+      @disable_ssl_peer_verification = options[:disable_ssl_peer_verification]
+      @disable_ssl_host_verification = options[:disable_ssl_host_verification]
+      @ssl_cert         = options[:ssl_cert]
+      @ssl_cert_type    = options[:ssl_cert_type]
+      @ssl_key          = options[:ssl_key]
+      @ssl_key_type     = options[:ssl_key_type]
+      @ssl_key_password = options[:ssl_key_password]
+      @ssl_cacert       = options[:ssl_cacert]
+      @ssl_capath       = options[:ssl_capath]
+      @ssl_version      = options[:ssl_version]
+      @verbose          = options[:verbose]
+      @username         = options[:username]
+      @password         = options[:password]
+      @auth_method      = options[:auth_method]
+
+      if @method == :post
+        @params = @params ? Typhoeus::Utils.escape_params(@params) : nil
+        @url = url
+      end
+
+      @parsed_uri = URI.parse(@url)
+
+      @on_complete      = nil
+      @after_complete   = nil
+      @handled_response = nil
+    end
+
+    LOCALHOST_ALIASES = %w[ localhost 127.0.0.1 0.0.0.0 ]
+
+    def localhost?
+      LOCALHOST_ALIASES.include?(parsed_uri.host)
+    end
+
+    def user_agent
+      headers['User-Agent']
+    end
+
+    def url
+      if @method == :post
+        @url
+      else
+        url = "#{@url}?#{params_string}"
+        url += "&#{URI.escape(@body)}" if @body
+        url.gsub("?&", "?").gsub(/\?$/, '')
+      end
+    end
+
+    def parsed_uri
+      @parsed_uri ||= URI.parse(@url)
+    end
+
+    def host
+      slash_location = @url.index('/', 8)
+      if slash_location
+        @url.slice(0, slash_location)
+      else
+        query_string_location = @url.index('?')
+        return query_string_location ? @url.slice(0, query_string_location) : @url
+      end
+    end
+
+    def host_domain
+      parsed_uri.host
+    end
+
+    def params_string
+      return nil unless params
+      traversal = Typhoeus::Utils.traverse_params_hash(params)
+      Typhoeus::Utils.traversal_to_param_string(traversal)
+    end
+
+    def on_complete(&block)
+      @on_complete = block
+    end
+
+    def on_complete=(proc)
+      @on_complete = proc
+    end
+
+    def after_complete(&block)
+      @after_complete = block
+    end
+
+    def after_complete=(proc)
+      @after_complete = proc
+    end
+
+    def call_handlers
+      if @on_complete
+        @handled_response = @on_complete.call(response)
+        call_after_complete
+      end
+    end
+
+    def call_after_complete
+      @after_complete.call(@handled_response) if @after_complete
+    end
+
+    def handled_response=(val)
+      @handled_response = val
+    end
+
+    def handled_response
+      @handled_response || response
+    end
+
+    def inspect
+      result = ":method => #{self.method.inspect},\n" <<
+               "\t:url => #{URI.parse(self.url).to_s}"
+      if self.body and !self.body.empty?
+        result << ",\n\t:body => #{self.body.inspect}"
+      end
+
+      if self.params and !self.params.empty?
+        result << ",\n\t:params => #{self.params.inspect}"
+      end
+
+      if self.headers and !self.headers.empty?
+        result << ",\n\t:headers => #{self.headers.inspect}"
+      end
+
+      result
+    end
+
+    def cache_key
+      Digest::SHA1.hexdigest(cache_key_basis || url)
+    end
+
+    def self.run(url, params)
+      r = new(url, params)
+      Typhoeus::Hydra.hydra.queue r
+      Typhoeus::Hydra.hydra.run
+      r.response
+    end
+
+    def self.get(url, params = {})
+      run(url, params.merge(:method => :get))
+    end
+
+    def self.post(url, params = {})
+      run(url, params.merge(:method => :post))
+    end
+
+    def self.put(url, params = {})
+      run(url, params.merge(:method => :put))
+    end
+
+    def self.delete(url, params = {})
+      run(url, params.merge(:method => :delete))
+    end
+
+    def self.head(url, params = {})
+      run(url, params.merge(:method => :head))
+    end
+
+  protected
+
+    # Return the important data needed to serialize this Request, except the
+    # `on_complete` and `after_complete` handlers, since they cannot be
+    # marshalled.
+    def marshal_dump
+      (instance_variables - ['@on_complete', '@after_complete', :@on_complete, :@after_complete]).map do |name|
+        [name, instance_variable_get(name)]
+      end
+    end
+
+    def marshal_load(attributes)
+      attributes.each { |name, value| instance_variable_set(name, value) }
+    end
+
+    def self.options
+      ACCESSOR_OPTIONS
+    end
+
+  private
+
+    def safe_to_i(value)
+      return value if value.is_a?(Fixnum)
+      return nil if value.nil?
+      return nil if value.respond_to?(:empty?) && value.empty?
+      value.to_i
+    end
+  end
+end

data/lib/typhoeus/response.rb

@@ -0,0 +1,122 @@
+module Typhoeus
+  class Response
+    attr_accessor :request, :mock
+    attr_reader :code, :headers, :body, :time,
+                :requested_url, :requested_remote_method,
+                :requested_http_method, :start_time,
+                :effective_url, :start_transfer_time,
+                :app_connect_time, :pretransfer_time,
+                :connect_time, :name_lookup_time,
+                :curl_return_code, :curl_error_message,
+                :primary_ip
+
+    attr_writer :headers_hash
+
+    def initialize(params = {})
+      @code                  = params[:code]
+      @curl_return_code      = params[:curl_return_code]
+      @curl_error_message    = params[:curl_error_message]
+      @status_message        = params[:status_message]
+      @http_version          = params[:http_version]
+      @headers               = params[:headers]
+      @body                  = params[:body]
+      @time                  = params[:time]
+      @requested_url         = params[:requested_url]
+      @requested_http_method = params[:requested_http_method]
+      @start_time            = params[:start_time]
+      @start_transfer_time   = params[:start_transfer_time]
+      @app_connect_time      = params[:app_connect_time]
+      @pretransfer_time      = params[:pretransfer_time]
+      @connect_time          = params[:connect_time]
+      @name_lookup_time      = params[:name_lookup_time]
+      @request               = params[:request]
+      @effective_url         = params[:effective_url]
+      @primary_ip            = params[:primary_ip]
+      @mock                  = params[:mock] || false  # default
+      @headers_hash          = Header.new(params[:headers_hash]) if params[:headers_hash]
+    end
+
+    # Returns true if this is a mock response.
+    def mock?
+      @mock
+    end
+
+    def headers
+      @headers ||= @headers_hash ? construct_header_string : ''
+    end
+
+    def headers_hash
+      @headers_hash ||= begin
+        headers.split("\n").map {|o| o.strip}.inject(Typhoeus::Header.new) do |hash, o|
+          if o.empty? || o =~ /^HTTP\/[\d\.]+/
+            hash
+          else
+            i = o.index(":") || o.size
+            key = o.slice(0, i)
+            value = o.slice(i + 1, o.size)
+            value = value.strip unless value.nil?
+            if hash.key? key
+              hash[key] = [hash[key], value].flatten
+            else
+              hash[key] = value
+            end
+
+            hash
+          end
+        end
+      end
+    end
+
+    def status_message
+      return @status_message if @status_message != nil
+
+      # HTTP servers can choose not to include the explanation to HTTP codes. The RFC
+      # states this (http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4):
+      # Except when responding to a HEAD request, the server SHOULD include an entity containing
+      # an explanation of the error situation [...]
+      # This means 'HTTP/1.1 404' is as valid as 'HTTP/1.1 404 Not Found' and we have to handle it.
+
+      # Regexp doc: http://rubular.com/r/eAr1oVYsVa
+      if first_header_line != nil and first_header_line[/\d{3} (.*)$/, 1] != nil
+        @status_message = first_header_line[/\d{3} (.*)$/, 1].chomp
+      else
+        @status_message = nil
+      end
+    end
+
+    def http_version
+      @http_version ||= first_header_line ? first_header_line[/HTTP\/(\S+)/, 1] : nil
+    end
+
+    def success?
+      @code >= 200 && @code < 300
+    end
+
+    def modified?
+      @code != 304
+    end
+
+    def timed_out?
+      curl_return_code == 28
+    end
+
+    private
+
+      def first_header_line
+        @first_header_line ||= @headers.to_s.split("\n").first
+      end
+
+      def construct_header_string
+        lines = ["HTTP/#{http_version} #{code} #{status_message}"]
+
+        @headers_hash.each do |key, values|
+          [values].flatten.each do |value|
+            lines << "#{key}: #{value}"
+          end
+        end
+
+        lines << '' << ''
+        lines.join("\r\n")
+      end
+  end
+end

data/lib/typhoeus/utils.rb

@@ -0,0 +1,58 @@
+module Typhoeus
+  module Utils
+    # Taken from Rack::Utils, 1.2.1 to remove Rack dependency.
+    def escape(s)
+      s.to_s.gsub(/([^ a-zA-Z0-9_.-]+)/u) {
+        '%'+$1.unpack('H2'*bytesize($1)).join('%').upcase
+      }.tr(' ', '+')
+    end
+    module_function :escape
+
+    def escape_params(params)
+      traverse_params_hash(params)[:params].inject({}) do |memo, (k, v)|
+        memo[escape(k)] = escape(v)
+        memo
+      end
+    end
+    module_function :escape_params
+
+    # Params are NOT escaped.
+    def traverse_params_hash(hash, result = nil, current_key = nil)
+      result = ParamProcessor.traverse_params_hash hash, result, current_key
+    end
+    module_function :traverse_params_hash
+
+    def traversal_to_param_string(traversal, escape = true)
+      traversal[:params].collect { |param|
+        escape ? "#{Typhoeus::Utils.escape(param[0])}=#{Typhoeus::Utils.escape(param[1])}" : "#{param[0]}=#{param[1]}"
+      }.join('&')
+    end
+    module_function :traversal_to_param_string
+
+    # Return the bytesize of String; uses String#size under Ruby 1.8 and
+    # String#bytesize under 1.9.
+    if ''.respond_to?(:bytesize)
+      def bytesize(string)
+        string.bytesize
+      end
+    else
+      def bytesize(string)
+        string.size
+      end
+    end
+    module_function :bytesize
+
+    # Return a byteslice from a string; uses String#[] under Ruby 1.8 and
+    # String#byteslice under 1.9.
+    if ''.respond_to?(:byteslice)
+      def byteslice(string, *args)
+        string.byteslice(*args)
+      end
+    else
+      def byteslice(string, *args)
+        string[*args]
+      end
+    end
+    module_function :byteslice
+  end
+end

data/lib/typhoeus/version.rb

@@ -0,0 +1,3 @@
+module Typhoeus
+  VERSION = '0.4.0'
+end

data/lib/typhoeus.rb

@@ -0,0 +1,57 @@
+require 'digest/sha2'
+require 'active_support/core_ext'
+
+require 'typhoeus/utils'
+require 'typhoeus/header'
+require 'typhoeus/curl'
+require 'typhoeus/easy'
+require 'typhoeus/form'
+require 'typhoeus/multi'
+require 'typhoeus/filter'
+require 'typhoeus/param_processor'
+require 'typhoeus/remote'
+require 'typhoeus/remote_proxy_object'
+require 'typhoeus/response'
+require 'typhoeus/request'
+require 'typhoeus/hydra'
+require 'typhoeus/hydra_mock'
+require 'typhoeus/version'
+
+module Typhoeus
+  def self.easy_object_pool
+    @easy_objects ||= []
+  end
+
+  def self.init_easy_object_pool
+    20.times do
+      easy_object_pool << Typhoeus::Easy.new
+    end
+  end
+
+  def self.release_easy_object(easy)
+    easy.reset
+    easy_object_pool << easy
+  end
+
+  def self.get_easy_object
+    if easy_object_pool.empty?
+      Typhoeus::Easy.new
+    else
+      easy_object_pool.pop
+    end
+  end
+
+  def self.add_easy_request(easy_object)
+    Thread.current[:curl_multi] ||= Typhoeus::Multi.new
+    Thread.current[:curl_multi].add(easy_object)
+  end
+
+  def self.perform_easy_requests
+    multi = Thread.current[:curl_multi]
+    start_time = Time.now
+    multi.easy_handles.each do |easy|
+      easy.start_time = start_time
+    end
+    multi.perform
+  end
+end

metadata

@@ -0,0 +1,179 @@
+--- !ruby/object:Gem::Specification
+name: dcu-typhoeus
+version: !ruby/object:Gem::Version
+  version: 0.4.0
+  prerelease: 
+platform: ruby
+authors:
+- David Balatero
+- Paul Dix
+- Hans Hasselberg
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-05-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ffi
+  requirement: &70327899192580 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70327899192580
+- !ruby/object:Gem::Dependency
+  name: mime-types
+  requirement: &70327899192100 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.18'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70327899192100
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: &70327899223600 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70327899223600
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: &70327899223120 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: *70327899223120
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: &70327899222640 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: *70327899222640
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &70327899222160 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: *70327899222160
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: &70327899221680 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :development
+  prerelease: false
+  version_requirements: *70327899221680
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70327899221200 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.10'
+  type: :development
+  prerelease: false
+  version_requirements: *70327899221200
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: &70327899220720 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :development
+  prerelease: false
+  version_requirements: *70327899220720
+description: Like a modern code version of the mythical beast with 100 serpent heads,
+  Typhoeus runs HTTP requests in parallel while cleanly encapsulating handling logic.
+email: hans.hasselberg@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/typhoeus/curl.rb
+- lib/typhoeus/easy/auth.rb
+- lib/typhoeus/easy/callbacks.rb
+- lib/typhoeus/easy/ffi_helper.rb
+- lib/typhoeus/easy/infos.rb
+- lib/typhoeus/easy/options.rb
+- lib/typhoeus/easy/proxy.rb
+- lib/typhoeus/easy/ssl.rb
+- lib/typhoeus/easy.rb
+- lib/typhoeus/filter.rb
+- lib/typhoeus/form.rb
+- lib/typhoeus/header.rb
+- lib/typhoeus/hydra/callbacks.rb
+- lib/typhoeus/hydra/connect_options.rb
+- lib/typhoeus/hydra/stubbing.rb
+- lib/typhoeus/hydra.rb
+- lib/typhoeus/hydra_mock.rb
+- lib/typhoeus/multi.rb
+- lib/typhoeus/param_processor.rb
+- lib/typhoeus/remote.rb
+- lib/typhoeus/remote_method.rb
+- lib/typhoeus/remote_proxy_object.rb
+- lib/typhoeus/request.rb
+- lib/typhoeus/response.rb
+- lib/typhoeus/utils.rb
+- lib/typhoeus/version.rb
+- lib/typhoeus.rb
+- CHANGELOG.md
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+homepage: https://github.com/dbalatero/typhoeus
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- /Users/krawek/Projects/OpenSource/typhoeus/lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.3.6
+requirements: []
+rubyforge_project: ! '[none]'
+rubygems_version: 1.8.11
+signing_key: 
+specification_version: 3
+summary: Parallel HTTP library on top of libcurl multi.
+test_files: []
+has_rdoc: