hhry-typhoeus 0.4.0

data/lib/typhoeus/easy/ssl.rb

@@ -0,0 +1,82 @@
+module Typhoeus
+  module EasyFu
+    module SSL
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      module ClassMethods
+        def valid_ssl_version(version)
+          Typhoeus::Easy::SSL_VERSIONS.key?(version.to_sym)
+        end
+      end
+
+      def ssl_version
+        @ssl_version
+      end
+
+      def ssl_version=(version)
+        raise "Invalid SSL version: '#{version}' supplied! Please supply one as listed in Typhoeus::Easy::SSL_VERSIONS" unless self.class.valid_ssl_version(version)
+        @ssl_version = version
+
+        set_option(:sslversion, Typhoeus::Easy::SSL_VERSIONS[version])
+      end
+
+      def disable_ssl_peer_verification
+        set_option(:verifypeer, 0)
+      end
+
+      def disable_ssl_host_verification
+        set_option(:verifyhost, 0)
+      end
+
+      # Set SSL certificate
+      # " The string should be the file name of your certificate. "
+      # The default format is "PEM" and can be changed with ssl_cert_type=
+      def ssl_cert=(cert)
+        set_option(:sslcert, cert)
+      end
+
+      # Set SSL certificate type
+      # " The string should be the format of your certificate. Supported formats are "PEM" and "DER" "
+      def ssl_cert_type=(cert_type)
+        raise "Invalid ssl cert type : '#{cert_type}'..." if cert_type and !%w(PEM DER p12).include?(cert_type)
+        set_option(:sslcerttype, cert_type)
+      end
+
+      # Set SSL Key file
+      # " The string should be the file name of your private key. "
+      # The default format is "PEM" and can be changed with ssl_key_type=
+      #
+      def ssl_key=(key)
+        set_option(:sslkey, key)
+      end
+
+      # Set SSL Key type
+      # " The string should be the format of your private key. Supported formats are "PEM", "DER" and "ENG". "
+      #
+      def ssl_key_type=(key_type)
+        raise "Invalid ssl key type : '#{key_type}'..." if key_type and !%w(PEM DER ENG).include?(key_type)
+        set_option(:sslkeytype, key_type)
+      end
+
+      def ssl_key_password=(key_password)
+        set_option(:keypasswd, key_password)
+      end
+
+      # Set SSL CACERT
+      # " File holding one or more certificates to verify the peer with. "
+      #
+      def ssl_cacert=(cacert)
+        set_option(:cainfo, cacert)
+      end
+
+      # Set CAPATH
+      # " directory holding multiple CA certificates to verify the peer with. The certificate directory must be prepared using the openssl c_rehash utility. "
+      #
+      def ssl_capath=(capath)
+        set_option(:capath, capath)
+      end
+    end
+  end
+end

data/lib/typhoeus/filter.rb

@@ -0,0 +1,28 @@
+module Typhoeus
+  class Filter
+    attr_reader :method_name
+    
+    def initialize(method_name, options = {})
+      @method_name = method_name
+      @options = options
+    end
+    
+    def apply_filter?(method_name)
+      if @options[:only]
+        if @options[:only].instance_of? Symbol
+          @options[:only] == method_name
+        else
+          @options[:only].include?(method_name)
+        end
+      elsif @options[:except]
+        if @options[:except].instance_of? Symbol
+          @options[:except] != method_name
+        else
+          !@options[:except].include?(method_name)
+        end
+      else
+        true
+      end
+    end
+  end
+end

data/lib/typhoeus/form.rb

@@ -0,0 +1,61 @@
+require 'mime/types'
+
+module Typhoeus
+  class Form
+    attr_accessor :params
+    attr_reader :first, :traversal
+
+    def initialize(params = {})
+      @params = params
+      @first = ::FFI::MemoryPointer.new(:pointer)
+      @last = ::FFI::MemoryPointer.new(:pointer)
+
+      ObjectSpace.define_finalizer(self, self.class.finalizer(self))
+    end
+
+    def self.finalizer(form)
+      proc { Curl.formfree(form.first) }
+    end
+
+    def traversal
+      @traversal ||= Typhoeus::Utils.traverse_params_hash(params)
+    end
+
+    def formadd_param(name, contents)
+      Curl.formadd(@first, @last,
+        :form_option, :copyname, :pointer, name,
+        :form_option, :namelength, :long, Utils.bytesize(name),
+        :form_option, :copycontents, :pointer, contents,
+        :form_option, :contentslength, :long, Utils.bytesize(contents),
+        :form_option, :end)
+    end
+    private :formadd_param
+
+    def formadd_file(name, filename, contenttype, file)
+      Curl.formadd(@first, @last,
+        :form_option, :copyname, :pointer, name,
+        :form_option, :namelength, :long, Utils.bytesize(name),
+        :form_option, :file, :string, file,
+        :form_option, :filename, :string, filename,
+        :form_option, :contenttype, :string, contenttype,
+        :form_option, :end)
+    end
+    private :formadd_file
+
+    def process!
+      # add params
+      traversal[:params].each { |p| formadd_param(p[0], p[1]) }
+
+      # add files
+      traversal[:files].each { |file_args| formadd_file(*file_args) }
+    end
+
+    def multipart?
+      !traversal[:files].empty?
+    end
+
+    def to_s
+      Typhoeus::Utils.traversal_to_param_string(traversal, false)
+    end
+  end
+end

data/lib/typhoeus/header.rb

@@ -0,0 +1,54 @@
+module Typhoeus
+  class Header < ::Hash
+    def initialize(constructor = {})
+      if constructor.is_a?(Hash)
+        super
+        update(constructor)
+      else
+        super(constructor)
+      end
+    end
+
+    def fetch(key, *extras)
+      super(convert_key(key), *extras)
+    end
+
+    def key?(key)
+      super(convert_key(key))
+    end
+
+    def [](key)
+      super(convert_key(key))
+    end
+
+    def []=(key, value)
+      super(convert_key(key), value)
+    end
+
+    def update(other_hash)
+      other_hash.each_pair do |key, value|
+        self[convert_key(key)] = value
+      end
+      self
+    end
+
+    alias_method :merge!, :update
+
+    def dup
+      self.class.new(Marshal.load(Marshal.dump(self)))
+    end
+
+    def merge(hash)
+      self.dup.update(hash)
+    end
+
+    def delete(key)
+      super(convert_key(key))
+    end
+
+  private
+    def convert_key(key)
+      key.to_s.split(/_|-/).map { |segment| segment.capitalize }.join("-")
+    end
+  end
+end

data/lib/typhoeus/hydra.rb

@@ -0,0 +1,246 @@
+require 'typhoeus/hydra/callbacks'
+require 'typhoeus/hydra/connect_options'
+require 'typhoeus/hydra/stubbing'
+
+module Typhoeus
+  class Hydra
+    include ConnectOptions
+    include Stubbing
+    extend Callbacks
+
+    def initialize(options = {})
+      @memoize_requests = true
+      @multi       = Multi.new
+      @easy_pool   = []
+      initial_pool_size = options[:initial_pool_size] || 10
+      @max_concurrency = options[:max_concurrency] || 200
+      initial_pool_size.times { @easy_pool << Easy.new }
+      @memoized_requests = {}
+      @retrieved_from_cache = {}
+      @queued_requests = []
+      @running_requests = 0
+
+      self.stubs = []
+      @active_stubs = []
+    end
+
+    def self.hydra
+      @hydra ||= new
+    end
+
+    def self.hydra=(val)
+      @hydra = val
+    end
+
+    #
+    # Abort the run on a best-effort basis.
+    #
+    # It won't abort the current burst of @max_concurrency requests,
+    # however it won't fire the rest of the queued requests so the run
+    # will be aborted as soon as possible...
+    #
+    def abort
+      @queued_requests.clear
+    end
+
+    def clear_cache_callbacks
+      @cache_setter = nil
+      @cache_getter = nil
+    end
+
+    def fire_and_forget
+      @queued_requests.each {|r| queue(r, false)}
+      @multi.fire_and_forget
+    end
+
+    def queue(request, obey_concurrency_limit = true)
+      return if assign_to_stub(request)
+
+      # At this point, we are running over live HTTP. Make sure we haven't
+      # disabled live requests.
+      check_allow_net_connect!(request)
+
+      if @running_requests >= @max_concurrency && obey_concurrency_limit
+        @queued_requests << request
+      else
+        if request.method == :get
+          if @memoize_requests && @memoized_requests.key?(request.url)
+            if response = @retrieved_from_cache[request.url]
+              request.response = response
+              request.call_handlers
+            else
+              @memoized_requests[request.url] << request
+            end
+          else
+            @memoized_requests[request.url] = [] if @memoize_requests
+            get_from_cache_or_queue(request)
+          end
+        else
+          get_from_cache_or_queue(request)
+        end
+      end
+    end
+
+    def run
+      while !@active_stubs.empty?
+        m = @active_stubs.first
+        while request = m.requests.shift
+          response = m.response
+          response.request = request
+          handle_request(request, response)
+        end
+        @active_stubs.delete(m)
+      end
+
+      @multi.perform
+    ensure
+      @multi.reset_easy_handles{|easy| release_easy_object(easy)}
+      @memoized_requests = {}
+      @retrieved_from_cache = {}
+      @running_requests = 0
+    end
+
+    def disable_memoization
+      @memoize_requests = false
+    end
+
+    def cache_getter(&block)
+      @cache_getter = block
+    end
+
+    def cache_setter(&block)
+      @cache_setter = block
+    end
+
+    def on_complete(&block)
+      @on_complete = block
+    end
+
+    def on_complete=(proc)
+      @on_complete = proc
+    end
+
+    private
+
+    def get_from_cache_or_queue(request)
+      if @cache_getter
+        val = @cache_getter.call(request)
+        if val
+          @retrieved_from_cache[request.url] = val
+          queue_next
+          handle_request(request, val, false)
+        else
+          @multi.add(get_easy_object(request))
+        end
+      else
+        @multi.add(get_easy_object(request))
+      end
+    end
+
+    def get_easy_object(request)
+      @running_requests += 1
+
+      easy = @easy_pool.pop || Easy.new
+      easy.verbose          = request.verbose
+      if request.username || request.password
+        auth = { :username => request.username, :password => request.password }
+        auth[:method] = request.auth_method if request.auth_method
+        easy.auth = auth
+      end
+
+      if request.proxy
+        proxy = { :server => request.proxy }
+        proxy[:type] = request.proxy_type if request.proxy_type
+        easy.proxy = proxy if request.proxy
+      end
+
+      if request.proxy_username || request.proxy_password
+        auth = { :username => request.proxy_username, :password => request.proxy_password }
+        auth[:method] = request.proxy_auth_method if request.proxy_auth_method
+        easy.proxy_auth = auth
+      end
+
+      easy.url          = request.url
+      easy.method       = request.method
+      easy.params       = request.params  if request.method == :post && request.params.present?
+      easy.headers      = request.headers if request.headers
+      easy.request_body = request.body    if request.method == :post && request.body.present?
+      easy.timeout      = request.timeout if request.timeout
+      easy.connect_timeout = request.connect_timeout if request.connect_timeout
+      easy.interface       = request.interface if request.interface
+      easy.follow_location = request.follow_location if request.follow_location
+      easy.max_redirects = request.max_redirects if request.max_redirects
+      easy.disable_ssl_peer_verification if request.disable_ssl_peer_verification
+      easy.disable_ssl_host_verification if request.disable_ssl_host_verification
+      easy.ssl_cert         = request.ssl_cert
+      easy.ssl_cert_type    = request.ssl_cert_type
+      easy.ssl_key          = request.ssl_key
+      easy.ssl_key_type     = request.ssl_key_type
+      easy.ssl_key_password = request.ssl_key_password
+      easy.ssl_cacert       = request.ssl_cacert
+      easy.ssl_capath       = request.ssl_capath
+      easy.ssl_version      = request.ssl_version || :default
+      easy.verbose          = request.verbose
+
+      easy.on_success do |easy|
+        queue_next
+        handle_request(request, response_from_easy(easy, request))
+        release_easy_object(easy)
+      end
+      easy.on_failure do |easy|
+        queue_next
+        handle_request(request, response_from_easy(easy, request))
+        release_easy_object(easy)
+      end
+      easy.set_headers
+      easy
+    end
+
+    def queue_next
+      @running_requests -= 1
+      queue(@queued_requests.shift) unless @queued_requests.empty?
+    end
+
+    def release_easy_object(easy)
+      easy.reset
+      @easy_pool.push easy
+    end
+
+    def handle_request(request, response, live_request = true)
+      request.response = response
+
+      self.class.run_global_hooks_for(:after_request_before_on_complete,
+                                      request)
+
+      if live_request && request.cache_timeout && @cache_setter
+        @cache_setter.call(request)
+      end
+      @on_complete.call(response) if @on_complete
+
+      request.call_handlers
+      if requests = @memoized_requests[request.url]
+        requests.each do |r|
+          r.response = response
+          r.call_handlers
+        end
+      end
+    end
+
+    def response_from_easy(easy, request)
+      Response.new(:code                => easy.response_code,
+                   :headers             => easy.response_header,
+                   :body                => easy.response_body,
+                   :time                => easy.total_time_taken,
+                   :start_transfer_time => easy.start_transfer_time,
+                   :app_connect_time    => easy.app_connect_time,
+                   :pretransfer_time    => easy.pretransfer_time,
+                   :connect_time        => easy.connect_time,
+                   :name_lookup_time    => easy.name_lookup_time,
+                   :effective_url       => easy.effective_url,
+                   :primary_ip          => easy.primary_ip,
+                   :curl_return_code => easy.curl_return_code,
+                   :curl_error_message => easy.curl_error_message,
+                   :request             => request)
+    end
+  end
+end