gravis-typhoeus 0.1.29

data/lib/typhoeus/remote.rb

@@ -0,0 +1,306 @@
+module Typhoeus
+  USER_AGENT = "Typhoeus - http://github.com/pauldix/typhoeus/tree/master"
+  
+  def self.included(base)
+    base.extend ClassMethods
+  end
+
+  class MockExpectedError < StandardError; end
+    
+  module ClassMethods
+    def allow_net_connect
+      @allow_net_connect = true if @allow_net_connect.nil?
+      @allow_net_connect
+    end
+
+    def allow_net_connect=(value)
+      @allow_net_connect = value
+    end
+    
+    def mock(method, args = {})
+      @remote_mocks ||= {}
+      @remote_mocks[method] ||= {}
+      args[:code]    ||= 200
+      args[:body]    ||= ""
+      args[:headers] ||= ""
+      args[:time]    ||= 0
+      url = args.delete(:url)
+      url ||= :catch_all
+      params = args.delete(:params)
+
+      key = mock_key_for(url, params)
+
+      @remote_mocks[method][key] = args
+    end
+
+    # Returns a key for a given URL and passed in 
+    # set of Typhoeus options to be used to store/retrieve
+    # a corresponding mock.
+    def mock_key_for(url, params = nil)
+      if url == :catch_all
+        url
+      else
+        key = url
+        if params and !params.empty?
+          key += flatten_and_sort_hash(params).to_s
+        end
+        key
+      end
+    end
+
+    def flatten_and_sort_hash(params)
+      params = params.dup
+
+      # Flatten any sub-hashes to a single string.
+      params.keys.each do |key|
+        if params[key].is_a?(Hash)
+          params[key] = params[key].sort_by { |k, v| k.to_s.downcase }.to_s
+        end
+      end
+
+      params.sort_by { |k, v| k.to_s.downcase }
+    end
+    
+    def get_mock(method, url, options)
+      return nil unless @remote_mocks
+      if @remote_mocks.has_key? method
+        extra_response_args = { :requested_http_method => method,
+                                :requested_url => url,
+                                :start_time => Time.now }
+        mock_key = mock_key_for(url, options[:params])
+        if @remote_mocks[method].has_key? mock_key
+          get_mock_and_run_handlers(method,
+                                    @remote_mocks[method][mock_key].merge(
+                                      extra_response_args),
+                                    options)
+        elsif @remote_mocks[method].has_key? :catch_all
+          get_mock_and_run_handlers(method,
+                                    @remote_mocks[method][:catch_all].merge(
+                                      extra_response_args),
+                                    options)
+        else
+          nil
+        end
+      else
+        nil
+      end
+    end
+
+    def enforce_allow_net_connect!(http_verb, url, params = nil)
+      if !allow_net_connect
+        message = "Real HTTP connections are disabled. Unregistered request: " <<
+                  "#{http_verb.to_s.upcase} #{url}\n" <<
+                  "  Try: mock(:#{http_verb}, :url => \"#{url}\""
+        if params
+          message << ",\n            :params => #{params.inspect}"
+        end
+
+        message << ")"
+
+        raise MockExpectedError, message
+      end
+    end
+
+    def check_expected_headers!(response_args, options)
+      missing_headers = {}
+
+      response_args[:expected_headers].each do |key, value|
+        if options[:headers].nil?
+          missing_headers[key] = [value, nil]
+        elsif ((options[:headers][key] && value != :anything) &&
+           options[:headers][key] != value)
+
+          missing_headers[key] = [value, options[:headers][key]]
+        end
+      end
+
+      unless missing_headers.empty?
+        raise headers_error_summary(response_args, options, missing_headers, 'expected')
+      end
+    end
+
+    def check_unexpected_headers!(response_args, options)
+      bad_headers = {}
+      response_args[:unexpected_headers].each do |key, value|
+        if (options[:headers][key] && value == :anything) ||
+           (options[:headers][key] == value)
+          bad_headers[key] = [value, options[:headers][key]]
+        end
+      end
+
+      unless bad_headers.empty?
+        raise headers_error_summary(response_args, options, bad_headers, 'did not expect')
+      end
+    end
+
+    def headers_error_summary(response_args, options, missing_headers, lead_in)
+      error = "#{lead_in} the following headers: #{response_args[:expected_headers].inspect}, but received: #{options[:headers].inspect}\n\n"
+      error   << "Differences:\n"
+      error   << "------------\n"
+      missing_headers.each do |key, values|
+        error << "  - #{key}: #{lead_in} #{values[0].inspect}, got #{values[1].inspect}\n"
+      end
+
+      error
+    end
+    private :headers_error_summary
+
+    def get_mock_and_run_handlers(method, response_args, options)
+      response = Response.new(response_args)
+     
+      if response_args.has_key? :expected_body
+        raise "#{method} expected body of \"#{response_args[:expected_body]}\" but received #{options[:body]}" if response_args[:expected_body] != options[:body]
+      end
+      
+      if response_args.has_key? :expected_headers
+        check_expected_headers!(response_args, options)
+      end
+
+      if response_args.has_key? :unexpected_headers
+        check_unexpected_headers!(response_args, options)
+      end
+
+      if response.code >= 200 && response.code < 300 && options.has_key?(:on_success)
+        response = options[:on_success].call(response)
+      elsif options.has_key?(:on_failure)
+        response = options[:on_failure].call(response)
+      end
+
+      encode_nil_response(response)
+    end
+       
+    [:get, :post, :put, :delete].each do |method|
+      line = __LINE__ + 2  # get any errors on the correct line num
+      code = <<-SRC
+        def #{method.to_s}(url, options = {})
+          mock_object = get_mock(:#{method.to_s}, url, options)
+          unless mock_object.nil?
+            decode_nil_response(mock_object)
+          else
+            enforce_allow_net_connect!(:#{method.to_s}, url, options[:params])
+            remote_proxy_object(url, :#{method.to_s}, options)
+          end
+        end
+      SRC
+      module_eval(code, "./lib/typhoeus/remote.rb", line)
+    end
+    
+    def remote_proxy_object(url, method, options)
+      easy = Typhoeus.get_easy_object
+      
+      easy.url                   = url
+      easy.method                = method
+      easy.headers               = options[:headers] if options.has_key?(:headers)
+      easy.headers["User-Agent"] = (options[:user_agent] || Typhoeus::USER_AGENT)
+      easy.params                = options[:params] if options[:params]
+      easy.request_body          = options[:body] if options[:body]
+      easy.timeout               = options[:timeout] if options[:timeout]
+      easy.set_headers
+      
+      proxy = Typhoeus::RemoteProxyObject.new(clear_memoized_proxy_objects, easy, options)
+      set_memoized_proxy_object(method, url, options, proxy)
+    end
+    
+    def remote_defaults(options)
+      @remote_defaults ||= {}
+      @remote_defaults.merge!(options) if options
+      @remote_defaults
+    end
+
+    # If we get subclassed, make sure that child inherits the remote defaults
+    # of the parent class.
+    def inherited(child)
+      child.__send__(:remote_defaults, @remote_defaults)
+    end
+    
+    def call_remote_method(method_name, args)
+      m = @remote_methods[method_name]
+      
+      base_uri = args.delete(:base_uri) || m.base_uri || ""
+
+      if args.has_key? :path
+        path = args.delete(:path)
+      else
+        path = m.interpolate_path_with_arguments(args)
+      end
+      path ||= ""
+      
+      http_method = m.http_method
+      url         = base_uri + path
+      options     = m.merge_options(args)
+      
+      # proxy_object = memoized_proxy_object(http_method, url, options)
+      # return proxy_object unless proxy_object.nil?
+      # 
+      # if m.cache_responses?
+      #   object = @cache.get(get_memcache_response_key(method_name, args))
+      #   if object
+      #     set_memoized_proxy_object(http_method, url, options, object)
+      #     return object
+      #   end
+      # end
+
+      proxy = memoized_proxy_object(http_method, url, options)
+      unless proxy
+        if m.cache_responses?
+          options[:cache] = @cache
+          options[:cache_key] = get_memcache_response_key(method_name, args)
+          options[:cache_timeout] = m.cache_ttl
+        end
+        proxy = send(http_method, url, options)
+      end
+      proxy
+    end
+    
+    def set_memoized_proxy_object(http_method, url, options, object)
+      @memoized_proxy_objects ||= {}
+      @memoized_proxy_objects["#{http_method}_#{url}_#{options.to_s}"] = object
+    end
+    
+    def memoized_proxy_object(http_method, url, options)
+      @memoized_proxy_objects ||= {}
+      @memoized_proxy_objects["#{http_method}_#{url}_#{options.to_s}"]
+    end
+    
+    def clear_memoized_proxy_objects
+      lambda { @memoized_proxy_objects = {} }
+    end
+
+    def get_memcache_response_key(remote_method_name, args)
+      result = "#{remote_method_name.to_s}-#{args.to_s}"
+      (Digest::SHA2.new << result).to_s
+    end
+    
+    def cache=(cache)
+      @cache = cache
+    end
+    
+    def define_remote_method(name, args = {})
+      @remote_defaults  ||= {}
+      args[:method]     ||= @remote_defaults[:method]
+      args[:on_success] ||= @remote_defaults[:on_success]
+      args[:on_failure] ||= @remote_defaults[:on_failure]
+      args[:base_uri]   ||= @remote_defaults[:base_uri]
+      args[:path]       ||= @remote_defaults[:path]
+      m = RemoteMethod.new(args)
+
+      @remote_methods ||= {}
+      @remote_methods[name] = m
+
+      class_eval <<-SRC
+        def self.#{name.to_s}(args = {})
+          call_remote_method(:#{name.to_s}, args)
+        end
+      SRC
+    end
+
+    private
+    def encode_nil_response(response)
+      response == nil ? :__nil__ : response
+    end
+
+    def decode_nil_response(response)
+      response == :__nil__ ? nil : response
+    end
+  end # ClassMethods
+end

data/lib/typhoeus/remote_method.rb

@@ -0,0 +1,108 @@
+module Typhoeus
+  class RemoteMethod
+    attr_accessor :http_method, :options, :base_uri, :path, :on_success, :on_failure, :cache_ttl
+    
+    def initialize(options = {})
+      @http_method       = options.delete(:method) || :get
+      @options           = options
+      @base_uri          = options.delete(:base_uri)
+      @path              = options.delete(:path)
+      @on_success        = options[:on_success]
+      @on_failure        = options[:on_failure]
+      @cache_responses   = options.delete(:cache_responses)
+      @memoize_responses = options.delete(:memoize_responses) || @cache_responses
+      @cache_ttl         = @cache_responses == true ? 0 : @cache_responses
+      @keys              = nil
+      
+      clear_cache
+    end
+    
+    def cache_responses?
+      @cache_responses
+    end
+    
+    def memoize_responses?
+      @memoize_responses
+    end
+    
+    def args_options_key(args, options)
+      "#{args.to_s}+#{options.to_s}"
+    end
+    
+    def calling(args, options)
+      @called_methods[args_options_key(args, options)] = true
+    end
+    
+    def already_called?(args, options)
+      @called_methods.has_key? args_options_key(args, options)
+    end
+    
+    def add_response_block(block, args, options)
+      @response_blocks[args_options_key(args, options)] << block
+    end
+    
+    def call_response_blocks(result, args, options)
+      key = args_options_key(args, options)
+      @response_blocks[key].each {|block| block.call(result)}
+      @response_blocks.delete(key)
+      @called_methods.delete(key)
+    end
+    
+    def clear_cache
+      @response_blocks  = Hash.new {|h, k| h[k] = []}
+      @called_methods   = {}      
+    end
+    
+    def merge_options(new_options)
+      merged = options.merge(new_options)
+      if options.has_key?(:params) && new_options.has_key?(:params)
+        merged[:params] = options[:params].merge(new_options[:params])
+      end
+      argument_names.each {|a| merged.delete(a)}
+      merged.delete(:on_success) if merged[:on_success].nil?
+      merged.delete(:on_failure) if merged[:on_failure].nil?
+      merged
+    end
+    
+    def interpolate_path_with_arguments(args)
+      interpolated_path = @path
+      argument_names.each do |arg|
+        interpolated_path = interpolated_path.gsub(":#{arg}", args[arg].to_s)
+      end
+      interpolated_path
+    end
+    
+    def argument_names
+      return @keys if @keys
+      pattern, keys = compile(@path)
+      @keys = keys.collect {|k| k.to_sym}
+    end
+    
+    # rippped from Sinatra. clean up stuff we don't need later
+    def compile(path)
+      path ||= ""
+      keys = []
+      if path.respond_to? :to_str
+        special_chars = %w{. + ( )}
+        pattern =
+          path.gsub(/((:\w+)|[\*#{special_chars.join}])/) do |match|
+            case match
+            when "*"
+              keys << 'splat'
+              "(.*?)"
+            when *special_chars
+              Regexp.escape(match)
+            else
+              keys << $2[1..-1]
+              "([^/?&#]+)"
+            end
+          end
+        [/^#{pattern}$/, keys]
+      elsif path.respond_to? :match
+        [path, keys]
+      else
+        raise TypeError, path
+      end
+    end
+  end
+end

data/lib/typhoeus/remote_proxy_object.rb

@@ -0,0 +1,48 @@
+module Typhoeus
+  class RemoteProxyObject
+    instance_methods.each { |m| undef_method m unless m =~ /^__|object_id/ }
+
+    def initialize(clear_memoized_store_proc, easy, options = {})
+      @clear_memoized_store_proc = clear_memoized_store_proc
+      @easy      = easy
+      @success   = options[:on_success]
+      @failure   = options[:on_failure]
+      @cache     = options.delete(:cache)
+      @cache_key = options.delete(:cache_key)
+      @timeout   = options.delete(:cache_timeout)
+      Typhoeus.add_easy_request(@easy)
+    end
+
+    def method_missing(sym, *args, &block)
+      unless @proxied_object
+        if @cache && @cache_key
+          @proxied_object = @cache.get(@cache_key) rescue nil
+        end
+
+        unless @proxied_object
+          Typhoeus.perform_easy_requests
+          response = Response.new(:code => @easy.response_code,
+                                  :headers => @easy.response_header,
+                                  :body => @easy.response_body,
+                                  :time => @easy.total_time_taken,
+                                  :requested_url => @easy.url,
+                                  :requested_http_method => @easy.method,
+                                  :start_time => @easy.start_time)
+          if @easy.response_code >= 200 && @easy.response_code < 300
+            Typhoeus.release_easy_object(@easy)
+            @proxied_object = @success.nil? ? response : @success.call(response)
+
+            if @cache && @cache_key
+              @cache.set(@cache_key, @proxied_object, @timeout)
+            end
+          else
+            @proxied_object = @failure.nil? ? response : @failure.call(response)
+          end
+         @clear_memoized_store_proc.call
+       end
+      end
+
+      @proxied_object.__send__(sym, *args, &block)
+    end
+  end
+end

data/lib/typhoeus/request.rb

@@ -0,0 +1,124 @@
+module Typhoeus
+  class Request
+    attr_accessor :method, :params, :body, :headers, :timeout, :user_agent, :response, :cache_timeout, :follow_location, :max_redirects, :proxy, :disable_ssl_peer_verification
+    attr_reader   :url
+
+    def initialize(url, options = {})
+      @method           = options[:method] || :get
+      @params           = options[:params]
+      @body             = options[:body]
+      @timeout          = options[:timeout]
+      @headers          = options[:headers] || {}
+      @user_agent       = options[:user_agent] || Typhoeus::USER_AGENT
+      @cache_timeout    = options[:cache_timeout]
+      @follow_location  = options[:follow_location]
+      @max_redirects    = options[:max_redirects]
+      @proxy            = options[:proxy]
+      @disable_ssl_peer_verification = options[:disable_ssl_peer_verification]
+
+      if @method == :post
+        @url = url
+      else
+        @url = @params ? "#{url}?#{params_string}" : url
+      end
+      @on_complete      = nil
+      @after_complete   = nil
+      @handled_response = nil
+    end
+
+    def host
+      slash_location = @url.index('/', 8)
+      if slash_location
+        @url.slice(0, slash_location)
+      else
+        query_string_location = @url.index('?')
+        return query_string_location ? @url.slice(0, query_string_location) : @url
+      end
+    end
+
+    def headers
+      @headers["User-Agent"] = @user_agent
+      @headers
+    end
+
+    def params_string
+      params.keys.sort { |a, b| a.to_s <=> b.to_s }.collect do |k|
+        value = params[k]
+        if value.is_a? Hash
+          value.keys.collect {|sk| Rack::Utils.escape("#{k}[#{sk}]") + "=" + Rack::Utils.escape(value[sk].to_s)}
+        elsif value.is_a? Array
+          key = Rack::Utils.escape(k.to_s)
+          value.collect { |v| "#{key}=#{Rack::Utils.escape(v.to_s)}" }.join('&')
+        else
+          "#{Rack::Utils.escape(k.to_s)}=#{Rack::Utils.escape(params[k].to_s)}"
+        end
+      end.flatten.join("&")
+    end
+
+    def on_complete(&block)
+      @on_complete = block
+    end
+
+    def on_complete=(proc)
+      @on_complete = proc
+    end
+
+    def after_complete(&block)
+      @after_complete = block
+    end
+
+    def after_complete=(proc)
+      @after_complete = proc
+    end
+
+    def call_handlers
+      if @on_complete
+        @handled_response = @on_complete.call(response)
+        call_after_complete
+      end
+    end
+
+    def call_after_complete
+       @after_complete.call(@handled_response) if @after_complete
+    end
+
+    def handled_response=(val)
+      @handled_response = val
+    end
+
+    def handled_response
+      @handled_response || response
+    end
+
+    def cache_key
+      Digest::SHA1.hexdigest(url)
+    end
+
+    def self.run(url, params)
+      r = new(url, params)
+      Typhoeus::Hydra.hydra.queue r
+      Typhoeus::Hydra.hydra.run
+      r.response
+    end
+
+    def self.get(url, params = {})
+      run(url, params.merge(:method => :get))
+    end
+
+    def self.post(url, params = {})
+      run(url, params.merge(:method => :post))
+    end
+
+    def self.put(url, params = {})
+      run(url, params.merge(:method => :put))
+    end
+
+    def self.delete(url, params = {})
+      run(url, params.merge(:method => :delete))
+    end
+
+    def self.head(url, params = {})
+      run(url, params.merge(:method => :head))
+    end
+  end
+end

data/lib/typhoeus/response.rb

@@ -0,0 +1,49 @@
+module Typhoeus
+  class Response
+    attr_accessor :request
+    attr_reader :code, :headers, :body, :time,
+                :requested_url, :requested_remote_method,
+                :requested_http_method, :start_time,
+                :effective_url
+
+    def initialize(params = {})
+      @code                  = params[:code]
+      @headers               = params[:headers]
+      @body                  = params[:body]
+      @time                  = params[:time]
+      @requested_url         = params[:requested_url]
+      @requested_http_method = params[:requested_http_method]
+      @start_time            = params[:start_time]
+      @request               = params[:request]
+      @effective_url         = params[:effective_url]
+    end
+
+    def headers_hash
+      headers.split("\n").map {|o| o.strip}.inject({}) do |hash, o|
+        if o.empty?
+          hash
+        else
+          i = o.index(":") || o.size
+          key = o.slice(0, i)
+          value = o.slice(i + 1, o.size)
+          value = value.strip unless value.nil?
+          if hash.has_key? key
+            hash[key] = [hash[key], value].flatten
+          else
+            hash[key] = value
+          end
+
+          hash
+        end
+      end
+    end
+
+    def success?
+      @code >= 200 && @code < 300
+    end
+
+    def modified?
+      @code != 304
+    end
+  end
+end

data/lib/typhoeus/service.rb

@@ -0,0 +1,20 @@
+module Typhoeus
+  class Service
+    def initialize(host, port)
+      @host = host
+      @port = port
+    end
+    
+    def get(resource, params)
+    end
+    
+    def put(resource, params)
+    end
+    
+    def post(resource, params)
+    end
+    
+    def delete(resource, params)
+    end
+  end
+end

data/lib/typhoeus.rb

@@ -0,0 +1,55 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__)) unless $LOAD_PATH.include?(File.dirname(__FILE__))
+
+require 'rack/utils'
+require 'digest/sha2'
+require 'typhoeus/easy'
+require 'typhoeus/multi'
+require 'typhoeus/native'
+require 'typhoeus/filter'
+require 'typhoeus/remote_method'
+require 'typhoeus/remote'
+require 'typhoeus/remote_proxy_object'
+require 'typhoeus/response'
+require 'typhoeus/request'
+require 'typhoeus/hydra'
+
+module Typhoeus
+  VERSION = File.read(File.dirname(__FILE__) + "/../VERSION").chomp
+
+  def self.easy_object_pool
+    @easy_objects ||= []
+  end
+
+  def self.init_easy_object_pool
+    20.times do
+      easy_object_pool << Typhoeus::Easy.new
+    end
+  end
+
+  def self.release_easy_object(easy)
+    easy.reset
+    easy_object_pool << easy
+  end
+
+  def self.get_easy_object
+    if easy_object_pool.empty?
+      Typhoeus::Easy.new
+    else
+      easy_object_pool.pop
+    end
+  end
+
+  def self.add_easy_request(easy_object)
+    Thread.current[:curl_multi] ||= Typhoeus::Multi.new
+    Thread.current[:curl_multi].add(easy_object)
+  end
+
+  def self.perform_easy_requests
+    multi = Thread.current[:curl_multi]
+    start_time = Time.now
+    multi.easy_handles.each do |easy|
+      easy.start_time = start_time
+    end
+    multi.perform
+  end
+end