pauldix-typhoeus 0.0.8

data/lib/typhoeus/easy.rb

@@ -0,0 +1,196 @@
+module Typhoeus
+  class Easy
+    attr_reader :response_body, :response_header, :method, :headers, :url
+    CURLINFO_STRING = 1048576
+    OPTION_VALUES = {
+      :CURLOPT_URL            => 10002,
+      :CURLOPT_HTTPGET        => 80,
+      :CURLOPT_HTTPPOST       => 10024,
+      :CURLOPT_UPLOAD         => 46,
+      :CURLOPT_CUSTOMREQUEST  => 10036,
+      :CURLOPT_POSTFIELDS     => 10015,
+      :CURLOPT_POSTFIELDSIZE  => 60,
+      :CURLOPT_USERAGENT      => 10018,
+      :CURLOPT_TIMEOUT_MS     => 155,
+      :CURLOPT_NOSIGNAL       => 99,
+      :CURLOPT_HTTPHEADER     => 10023,
+      :CURLOPT_FOLLOWLOCATION => 52
+    }
+    INFO_VALUES = {
+      :CURLINFO_RESPONSE_CODE => 2097154,
+      :CURLINFO_TOTAL_TIME    => 3145731
+    }
+    
+    def initialize
+      @method = :get
+      @post_dat_set = nil
+      @headers = {}
+    end
+    
+    def total_time_taken
+      get_info_double(INFO_VALUES[:CURLINFO_TOTAL_TIME])
+    end
+    
+    def response_code
+      get_info_long(INFO_VALUES[:CURLINFO_RESPONSE_CODE])
+    end
+    
+    def follow_location=(boolean)
+      if boolean
+        set_option(OPTION_VALUES[:CURLOPT_FOLLOWLOCATION], 1)
+      else
+        set_option(OPTION_VALUES[:CURLOPT_FOLLOWLOCATION], 0)
+      end
+    end
+    
+    def timeout=(milliseconds)
+      @timeout = milliseconds
+      set_option(OPTION_VALUES[:CURLOPT_NOSIGNAL], 1)
+      set_option(OPTION_VALUES[:CURLOPT_TIMEOUT_MS], milliseconds)
+    end
+    
+    def timed_out?
+      @timeout && total_time_taken > @timeout && response_code == 0
+    end
+    
+    def request_body=(request_body)
+      @request_body = request_body
+      if @method == :put
+        easy_set_request_body(@request_body)
+        headers["Transfer-Encoding"] = ""
+        headers["Expect"] = ""
+      else
+        self.post_data = request_body
+      end
+    end
+    
+    def user_agent=(user_agent)
+      set_option(OPTION_VALUES[:CURLOPT_USERAGENT], user_agent)
+    end
+          
+    def url=(url)
+      @url = url
+      set_option(OPTION_VALUES[:CURLOPT_URL], url)
+    end
+    
+    def method=(method)
+      @method = method
+      if method == :get
+        set_option(OPTION_VALUES[:CURLOPT_HTTPGET], 1)
+      elsif method == :post
+        set_option(OPTION_VALUES[:CURLOPT_HTTPPOST], 1)
+        self.post_data = ""
+      elsif method == :put
+        set_option(OPTION_VALUES[:CURLOPT_UPLOAD], 1)
+      else
+        set_option(OPTION_VALUES[:CURLOPT_CUSTOMREQUEST], "DELETE")
+      end
+    end
+    
+    def post_data=(data)
+      @post_data_set = true
+      set_option(OPTION_VALUES[:CURLOPT_POSTFIELDS], data)
+      set_option(OPTION_VALUES[:CURLOPT_POSTFIELDSIZE], data.length)
+    end
+    
+    def params=(params)
+      params_string = params.keys.collect do |k|
+        value = params[k]
+        if value.is_a? Hash
+          value.keys.collect {|sk| CGI.escape("#{k}[#{sk}]") + "=" + CGI.escape(value[sk].to_s)}
+        else
+          "#{CGI.escape(k.to_s)}=#{CGI.escape(params[k].to_s)}"
+        end
+      end.flatten.join("&")
+      
+      if method == :get
+        self.url = "#{url}?#{params_string}"
+      elsif method == :post
+        self.post_data = params_string
+      end
+    end
+    
+    def set_option(option, value)
+      if value.class == String
+        easy_setopt_string(option, value)
+      else
+        easy_setopt_long(option, value)
+      end
+    end
+    
+    def perform
+      headers.each_pair do |key, value|
+        easy_add_header("#{key}: #{value}")
+      end
+      easy_set_headers() unless headers.empty?
+      easy_perform()
+      response_code()
+    end
+    
+    # gets called when finished and response code is 200-299
+    def success
+      @success.call(self) if @success
+    end
+    
+    def on_success(&block)
+      @success = block
+    end
+    
+    def on_success=(block)
+      @success = block
+    end
+    
+    # gets called when finished and response code is 300-599
+    def failure
+      @failure.call(self) if @failure
+    end
+    
+    def on_failure(&block)
+      @failure = block
+    end
+    
+    def on_failure=(block)
+      @failure = block
+    end
+
+    def retries
+      @retries ||= 0
+    end
+    
+    def increment_retries
+      @retries ||= 0
+      @retries += 1
+    end
+    
+    def max_retries
+      @max_retries ||= 40
+    end
+    
+    def max_retries?
+      retries >= max_retries
+    end
+    
+    def reset
+      @response_code = 0
+      @response_header = ""
+      @response_body = ""
+      easy_reset()
+    end
+    
+    def get_info_string(option)
+      easy_getinfo_string(option)
+    end
+    
+    def get_info_long(option)
+      easy_getinfo_long(option)
+    end
+    
+    def get_info_double(option)
+      easy_getinfo_double(option)
+    end
+    
+    def curl_version
+      version
+    end
+  end
+end

data/lib/typhoeus/filter.rb

@@ -0,0 +1,28 @@
+module Typhoeus
+  class Filter
+    attr_reader :method_name
+    
+    def initialize(method_name, options = {})
+      @method_name = method_name
+      @options = options
+    end
+    
+    def apply_filter?(method_name)
+      if @options[:only]
+        if @options[:only].instance_of? Symbol
+          @options[:only] == method_name
+        else
+          @options[:only].include?(method_name)
+        end
+      elsif @options[:except]
+        if @options[:except].instance_of? Symbol
+          @options[:except] != method_name
+        else
+          !@options[:except].include?(method_name)
+        end
+      else
+        true
+      end
+    end
+  end
+end

data/lib/typhoeus/multi.rb

@@ -0,0 +1,21 @@
+module Typhoeus
+  class Multi
+    def remove(easy)
+      multi_remove_handle(easy)
+    end
+    
+    def add(easy)
+      multi_add_handle(easy)
+    end
+    
+    def perform()
+      while active_handle_count > 0 do
+        multi_perform
+      end
+    end
+    
+    def cleanup()
+      multi_cleanup
+    end
+  end
+end

data/lib/typhoeus/remote.rb

@@ -0,0 +1,126 @@
+module Typhoeus
+  USER_AGENT = "Typhoeus - http://github.com/pauldix/typhoeus/tree/master"
+  
+  def self.included(base)
+    base.extend ClassMethods
+  end
+    
+  module ClassMethods
+    def get(url, options = {})
+      remote_proxy_object(url, :get, options)
+    end
+    
+    def post(url, options = {}, &block)
+      remote_proxy_object(url, :post, options)
+    end
+
+    def put(url, options = {}, &block)
+      remote_proxy_object(url, :put, options)
+    end
+    
+    def delete(url, options = {}, &block)
+      remote_proxy_object(url, :delete, options)
+    end
+    
+    def remote_proxy_object(url, method, options)
+      easy = Typhoeus.get_easy_object
+      
+      easy.url                   = url
+      easy.method                = method
+      easy.headers["User-Agent"] = (options[:user_agent] || Typhoeus::USER_AGENT)
+      easy.params                = options[:params] if options[:params]
+      easy.request_body          = options[:body] if options[:body]
+      easy.timeout               = options[:timeout] if options[:timeout]
+      
+      easy
+      
+      proxy = Typhoeus::RemoteProxyObject.new(clear_memoized_proxy_objects, easy, options)
+      set_memoized_proxy_object(method, url, options, proxy)
+    end
+    
+    def remote_defaults(options)
+      @remote_defaults = options
+    end
+    
+    def call_remote_method(method_name, args)
+      m = @remote_methods[method_name]
+      
+      base_uri = args.delete(:base_uri) || m.base_uri || ""
+
+      if args.has_key? :path
+        path = args.delete(:path)
+      else
+        path = m.interpolate_path_with_arguments(args)
+      end
+      path ||= ""
+      
+      http_method = m.http_method
+      url         = base_uri + path
+      options     = m.merge_options(args)
+      
+      # proxy_object = memoized_proxy_object(http_method, url, options)
+      # return proxy_object unless proxy_object.nil?
+      # 
+      # if m.cache_responses?
+      #   object = @cache.get(get_memcache_response_key(method_name, args))
+      #   if object
+      #     set_memoized_proxy_object(http_method, url, options, object)
+      #     return object
+      #   end
+      # end
+      
+      proxy = memoized_proxy_object(http_method, url, options)
+      unless proxy
+        if m.cache_responses?
+          options[:cache] = @cache
+          options[:cache_key] = get_memcache_response_key(method_name, args)
+          options[:cache_timeout] = m.cache_ttl
+        end
+        proxy = send(http_method, url, options)
+      end
+      proxy
+    end
+    
+    def set_memoized_proxy_object(http_method, url, options, object)
+      @memoized_proxy_objects ||= {}
+      @memoized_proxy_objects["#{http_method}_#{url}_#{options.to_s}"] = object
+    end
+    
+    def memoized_proxy_object(http_method, url, options)
+      @memoized_proxy_objects ||= {}
+      @memoized_proxy_objects["#{http_method}_#{url}_#{options.to_s}"]
+    end
+    
+    def clear_memoized_proxy_objects
+      lambda { @memoized_proxy_objects = {} }
+    end
+
+    def get_memcache_response_key(remote_method_name, args)
+      result = "#{remote_method_name.to_s}-#{args.to_s}"
+      (Digest::SHA2.new << result).to_s
+    end
+    
+    def cache=(cache)
+      @cache = cache
+    end
+    
+    def define_remote_method(name, args = {})
+      @remote_defaults  ||= {}
+      args[:method]     ||= @remote_defaults[:method]
+      args[:on_success] ||= @remote_defaults[:on_success]
+      args[:on_failure] ||= @remote_defaults[:on_failure]
+      args[:base_uri]   ||= @remote_defaults[:base_uri]
+      args[:path]       ||= @remote_defaults[:path]
+      m = RemoteMethod.new(args)
+
+      @remote_methods ||= {}
+      @remote_methods[name] = m
+
+      class_eval <<-SRC
+        def self.#{name.to_s}(args = {})
+          call_remote_method(:#{name.to_s}, args)
+        end
+      SRC
+    end
+  end # ClassMethods
+end

data/lib/typhoeus/remote_method.rb

@@ -0,0 +1,107 @@
+module Typhoeus
+  class RemoteMethod
+    attr_accessor :http_method, :options, :base_uri, :path, :on_success, :on_failure, :cache_ttl
+    
+    def initialize(options = {})
+      @http_method       = options.delete(:method) || :get
+      @options           = options
+      @base_uri          = options.delete(:base_uri)
+      @path              = options.delete(:path)
+      @on_success        = options[:on_success]
+      @on_failure        = options[:on_failure]
+      @cache_responses   = options.delete(:cache_responses)
+      @memoize_responses = options.delete(:memoize_responses) || @cache_responses
+      @cache_ttl         = @cache_responses == true ? 0 : @cache_responses
+      
+      clear_cache
+    end
+    
+    def cache_responses?
+      @cache_responses
+    end
+    
+    def memoize_responses?
+      @memoize_responses
+    end
+    
+    def args_options_key(args, options)
+      "#{args.to_s}+#{options.to_s}"
+    end
+    
+    def calling(args, options)
+      @called_methods[args_options_key(args, options)] = true
+    end
+    
+    def already_called?(args, options)
+      @called_methods.has_key? args_options_key(args, options)
+    end
+    
+    def add_response_block(block, args, options)
+      @response_blocks[args_options_key(args, options)] << block
+    end
+    
+    def call_response_blocks(result, args, options)
+      key = args_options_key(args, options)
+      @response_blocks[key].each {|block| block.call(result)}
+      @response_blocks.delete(key)
+      @called_methods.delete(key)
+    end
+    
+    def clear_cache
+      @response_blocks  = Hash.new {|h, k| h[k] = []}
+      @called_methods   = {}      
+    end
+    
+    def merge_options(new_options)
+      merged = options.merge(new_options)
+      if options.has_key?(:params) && new_options.has_key?(:params)
+        merged[:params] = options[:params].merge(new_options[:params])
+      end
+      argument_names.each {|a| merged.delete(a)}
+      merged.delete(:on_success) if merged[:on_success].nil?
+      merged.delete(:on_failure) if merged[:on_failure].nil?
+      merged
+    end
+    
+    def interpolate_path_with_arguments(args)
+      interpolated_path = @path
+      argument_names.each do |arg|
+        interpolated_path = interpolated_path.gsub(":#{arg}", args[arg].to_s)
+      end
+      interpolated_path
+    end
+    
+    def argument_names
+      return @keys if @keys
+      pattern, keys = compile(@path)
+      @keys = keys.collect {|k| k.to_sym}
+    end
+    
+    # rippped from Sinatra. clean up stuff we don't need later
+    def compile(path)
+      path ||= ""
+      keys = []
+      if path.respond_to? :to_str
+        special_chars = %w{. + ( )}
+        pattern =
+          path.gsub(/((:\w+)|[\*#{special_chars.join}])/) do |match|
+            case match
+            when "*"
+              keys << 'splat'
+              "(.*?)"
+            when *special_chars
+              Regexp.escape(match)
+            else
+              keys << $2[1..-1]
+              "([^/?&#]+)"
+            end
+          end
+        [/^#{pattern}$/, keys]
+      elsif path.respond_to? :match
+        [path, keys]
+      else
+        raise TypeError, path
+      end
+    end
+  end
+end

data/lib/typhoeus/remote_proxy_object.rb

@@ -0,0 +1,42 @@
+module Typhoeus
+  class RemoteProxyObject
+    instance_methods.each { |m| undef_method m unless m =~ /^__/ }
+    
+    def initialize(clear_memoized_store_proc, easy, options = {})
+      @clear_memoized_store_proc = clear_memoized_store_proc
+      @easy      = easy
+      @success   = options[:on_success]
+      @failure   = options[:on_failure]
+      @cache     = options.delete(:cache)
+      @cache_key = options.delete(:cache_key)
+      @timeout   = options.delete(:cache_timeout)
+      Typhoeus.add_easy_request(@easy)
+    end
+    
+    def method_missing(sym, *args, &block)
+      unless @proxied_object
+        if @cache && @cache_key
+          @proxied_object = @cache.get(@cache_key)
+        end
+        
+        unless @proxied_object
+          Typhoeus.perform_easy_requests
+          response = Response.new(@easy.response_code, @easy.response_header, @easy.response_body, @easy.total_time_taken)
+          if @easy.response_code >= 200 && @easy.response_code < 300
+            Typhoeus.release_easy_object(@easy)
+            @proxied_object = @success.nil? ? response : @success.call(response)
+            
+            if @cache && @cache_key
+              @cache.set(@cache_key, @proxied_object, @timeout)
+            end
+          else
+            @proxied_object = @failure.nil? ? response : @failure.call(response)
+          end
+         @clear_memoized_store_proc.call
+       end
+      end
+      
+      @proxied_object.__send__(sym, *args, &block)
+    end
+  end
+end

data/lib/typhoeus/response.rb

@@ -0,0 +1,12 @@
+module Typhoeus
+  class Response
+    attr_reader :code, :headers, :body, :time
+    
+    def initialize(response_code, response_headers, response_body, request_time)
+      @code    = response_code
+      @headers = response_headers
+      @body    = response_body
+      @time    = request_time
+    end
+  end
+end

data/lib/typhoeus.rb

@@ -0,0 +1,48 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__)) unless $LOAD_PATH.include?(File.dirname(__FILE__))
+
+require 'cgi'
+require 'digest/sha2'
+require 'typhoeus/easy'
+require 'typhoeus/multi'
+require 'typhoeus/native'
+require 'typhoeus/filter'
+require 'typhoeus/remote_method'
+require 'typhoeus/remote'
+require 'typhoeus/remote_proxy_object'
+require 'typhoeus/response'
+
+module Typhoeus
+  VERSION = "0.0.8"
+  
+  def self.easy_object_pool
+    @easy_objects ||= []
+  end
+  
+  def self.init_easy_object_pool
+    20.times do
+      easy_object_pool << Typhoeus::Easy.new
+    end
+  end
+  
+  def self.release_easy_object(easy)
+    easy.reset
+    easy_object_pool << easy
+  end
+  
+  def self.get_easy_object
+    if easy_object_pool.empty?
+      Typhoeus::Easy.new
+    else
+      easy_object_pool.pop
+    end
+  end
+  
+  def self.add_easy_request(easy_object)
+    Thread.current[:curl_multi] ||= Typhoeus::Multi.new
+    Thread.current[:curl_multi].add(easy_object)
+  end
+  
+  def self.perform_easy_requests
+    Thread.current[:curl_multi].perform
+  end
+end

data/spec/servers/delay_fixture_server.rb

@@ -0,0 +1,66 @@
+# this server simply accepts requests and blocks for a passed in interval before returning a passed in reqeust value to
+# the client
+require 'rubygems'
+require 'eventmachine'
+require 'evma_httpserver'
+ 
+class DelayFixtureServer  < EventMachine::Connection
+  include EventMachine::HttpServer
+ 
+  def process_http_request
+    EventMachine.stop if ENV["PATH_INFO"] == "/die"
+    puts "got a request #{ENV['PATH_INFO']}"
+    resp = EventMachine::DelegatedHttpResponse.new( self )
+    
+    # Block which fulfills the request
+    operation = proc do
+      sleep DelayFixtureServer.response_delay
+
+      resp.status = 200
+      resp.content = "whatever"
+    end
+
+    # Callback block to execute once the request is fulfilled
+    callback = proc do |res|
+      resp.send_response
+    end
+
+    # Let the thread pool (20 Ruby threads) handle request
+    EM.defer(operation, callback)
+  end
+  
+  def self.response_fixture
+    @response_fixture ||= ""
+  end
+  
+  def self.response_fixture=(val)
+    @response_fixture = val
+  end
+
+  def self.response_delay
+    @response_delay ||= 0
+  end
+
+  def self.response_delay=(val)
+    @response_delay = val
+  end
+  
+  def self.reponse_number
+    @response_number
+  end
+  
+  def self.response_number=(val)
+    @response_number = val
+  end
+end
+
+port = (ARGV[0] || 3000).to_i
+
+DelayFixtureServer.response_delay   = 0.5
+DelayFixtureServer.response_number = 0
+#DelayFixtureServer.response_fixture = File.read(File.dirname(__FILE__) + "/../fixtures/result_set.xml")
+
+EventMachine::run {
+  EventMachine.epoll
+  EventMachine::start_server("0.0.0.0", port, DelayFixtureServer)
+}

data/spec/servers/method_server.rb

@@ -0,0 +1,51 @@
+# this server simply is for testing out the different http methods. it echoes back the passed in info
+require 'rubygems'
+require 'eventmachine'
+require 'evma_httpserver'
+ 
+class MethodServer  < EventMachine::Connection
+  include EventMachine::HttpServer
+ 
+  def process_http_request
+    EventMachine.stop if ENV["PATH_INFO"] == "/die"
+    
+    resp = EventMachine::DelegatedHttpResponse.new( self )
+    
+    # Block which fulfills the request
+    operation = proc do
+      sleep MethodServer.sleep_time
+      resp.status = 200
+      resp.content = request_params + "\n#{@http_post_content}"
+    end
+
+    # Callback block to execute once the request is fulfilled
+    callback = proc do |res|
+      resp.send_response
+    end
+
+    # Let the thread pool (20 Ruby threads) handle request
+    EM.defer(operation, callback)
+  end
+  
+  def request_params
+    %w( PATH_INFO QUERY_STRING HTTP_COOKIE IF_NONE_MATCH CONTENT_TYPE REQUEST_METHOD REQUEST_URI ).collect do |param|
+      "#{param}=#{ENV[param]}"
+    end.join("\n")
+  end
+  
+  def self.sleep_time=(val)
+    @sleep_time = val
+  end
+  
+  def self.sleep_time
+    @sleep_time || 0
+  end
+end
+# 
+# port = (ARGV[0] || 3000).to_i
+# #Process.fork do
+#   EventMachine::run {
+#     EventMachine.epoll
+#     EventMachine::start_server("0.0.0.0", port, MethodServer)
+#   }
+# #end

data/spec/spec.opts

@@ -0,0 +1,2 @@
+--diff
+--color