thrift_client 0.4.7 → 0.5.0

data/CHANGELOG

@@ -1,3 +1,7 @@
+v0.5.0 Add support for wrapping exception, so that Thrift::Foo can become Greeter::Foo.
+Make server_retry_period work the way you expect.
+Better bookkeeping around marking servers as dead.
+
 v0.4.7 fix thrift gem dependency
 
 v0.4.6 Add support for oneway methods.

data/lib/thrift_client/abstract_thrift_client.rb

@@ -1,20 +1,54 @@
 class AbstractThriftClient
 
+  class Server
+    attr_reader :connection_string, :marked_down_at
+
+    def initialize(connection_string)
+      @connection_string = connection_string
+    end
+    alias to_s connection_string
+
+    def mark_down!
+      @marked_down_at = Time.now
+    end
+  end
+
+  DISCONNECT_ERRORS = [
+    IOError,
+    Thrift::Exception,
+    Thrift::ApplicationException,
+    Thrift::TransportException
+  ]
+
+  DEFAULT_WRAPPED_ERRORS = [
+    Thrift::ApplicationException,
+    Thrift::TransportException,
+  ]
+
   DEFAULTS = {
     :protocol => Thrift::BinaryProtocol,
     :protocol_extra_params => [],
     :transport => Thrift::Socket,
     :transport_wrapper => Thrift::FramedTransport,
     :raise => true,
-    :defaults => {}
-  }.freeze
+    :defaults => {},
+    :exception_classes => DISCONNECT_ERRORS,
+    :retries => 0,
+    :server_retry_period => 1,
+    :server_max_requests => nil,
+    :retry_overrides => {},
+    :wrapped_exception_classes => DEFAULT_WRAPPED_ERRORS,
+    :timeout => 1,
+    :timeout_overrides => {}
+  }
 
   attr_reader :client, :client_class, :current_server, :server_list, :options, :client_methods
 
   def initialize(client_class, servers, options = {})
     @options = DEFAULTS.merge(options)
+    @options[:server_retry_period] ||= 0
     @client_class = client_class
-    @server_list = Array(servers)
+    @server_list = Array(servers).collect{|s| Server.new(s)}.sort_by { rand }
     @current_server = @server_list.first
 
     @client_methods = []
@@ -24,6 +58,15 @@ class AbstractThriftClient
         @client_methods << $1
       end
     end
+    @request_count = 0
+    @options[:wrapped_exception_classes].each do |exception_klass|
+      name = exception_klass.to_s.split('::').last
+      begin
+        @client_class.const_get(name)
+      rescue NameError
+        @client_class.const_set(name, Class.new(exception_klass))
+      end
+    end
   end
 
   def inspect
@@ -34,180 +77,91 @@ class AbstractThriftClient
   # called as the connection will be made on the first RPC method
   # call.
   def connect!
-    @connection = Connection::Factory.create(@options[:transport], @options[:transport_wrapper], @current_server, @options[:timeout])
+    @current_server = next_live_server
+    @connection = Connection::Factory.create(@options[:transport], @options[:transport_wrapper], @current_server.connection_string, @options[:timeout])
     @connection.connect!
     @client = @client_class.new(@options[:protocol].new(@connection.transport, *@options[:protocol_extra_params]))
-  rescue Thrift::TransportException, Errno::ECONNREFUSED => e
-    @transport.close rescue nil
-    raise e
   end
 
   def disconnect!
-    @connection.close rescue nil
+    @connection.close rescue nil #TODO
     @client = nil
     @current_server = nil
+    @request_count = 0
   end
 
   private
-  def handled_proxy(method_name, *args)
-    proxy(method_name, *args)
-  rescue Exception => e
-    handle_exception(e, method_name, args)
-  end
-
-  def post_connect(method_name); end
-
-  def proxy(method_name, *args)
-    connect! unless @client
-    post_connect(method_name)
-    send_rpc(method_name, *args)
-  end
-
-  def send_rpc(method_name, *args)
-    @client.send(method_name, *args)
-  end
-
-  def disconnect_on_error!
-    @connection.close rescue nil
-    @client = nil
-    @current_server = nil
-  end
-
-  def handle_exception(e, method_name, args=nil)
-    raise e if @options[:raise]
-    @options[:defaults][method_name.to_sym]
-  end
-
-  module RetryingThriftClient
-    DISCONNECT_ERRORS = [
-                         IOError,
-                         Thrift::Exception,
-                         Thrift::ProtocolException,
-                         Thrift::ApplicationException,
-                         Thrift::TransportException
-                        ].freeze
-
-    RETRYING_DEFAULTS = {
-      :exception_classes => DISCONNECT_ERRORS,
-      :randomize_server_list => true,
-      :retries => 0,
-      :server_retry_period => 1,
-      :server_max_requests => nil,
-      :retry_overrides => {}
-    }.freeze
-
-    def initialize(client_class, servers, options = {})
-      super
-      @options = RETRYING_DEFAULTS.merge(@options) # @options is set by super
-      @retries = @options[:retries]
-      @request_count = 0
-      @max_requests = @options[:server_max_requests]
-      @retry_period = @options[:server_retry_period]
-      rebuild_live_server_list!
-    end
-
-    def connect!
-      @current_server = next_server
-      super
-    rescue Thrift::TransportException, Errno::ECONNREFUSED
-      retry
-    end
-
-    def disconnect!
-      # Keep live servers in the list if we have a retry period. Otherwise,
-      # always eject, because we will always re-add them.
-      if @retry_period && @current_server
-        @live_server_list.unshift(@current_server)
-      end
-
-      super()
-      @request_count = 0
-    end
-
-    private
 
-    def next_server
-      if @retry_period
-        rebuild_live_server_list! if Time.now > @last_rebuild + @retry_period
-        raise ThriftClient::NoServersAvailable, "No live servers in #{@server_list.inspect} since #{@last_rebuild.inspect}." if @live_server_list.empty?
-      elsif @live_server_list.empty?
-        rebuild_live_server_list!
+  def next_live_server
+    @server_index ||= 0
+    @server_list.length.times do |i|
+      cur = (1 + @server_index + i) % @server_list.length
+      if !@server_list[cur].marked_down_at || (@server_list[cur].marked_down_at + @options[:server_retry_period] <= Time.now)
+        @server_index = cur
+        return @server_list[cur]
       end
-      @live_server_list.pop
     end
+    raise ThriftClient::NoServersAvailable, "No live servers in #{@server_list.inspect} since #{@last_rebuild.inspect}."
+  end
 
-    def rebuild_live_server_list!
-      @last_rebuild = Time.now
-      if @options[:randomize_server_list]
-        @live_server_list = @server_list.sort_by { rand }
-      else
-        @live_server_list = @server_list.dup
+  def handled_proxy(method_name, *args)
+    disconnect_on_max! if @options[:server_max_requests] && @request_count >= @options[:server_max_requests]
+    begin
+      connect! unless @client
+      if has_timeouts?
+        @client.timeout = @options[:timeout_overrides][method_name.to_sym] || @options[:timeout]
       end
-    end
-
-    def handled_proxy(method_name, *args)
-      disconnect_on_max! if @max_requests and @request_count >= @max_requests
-      super
-    end
-
-    def proxy(method_name, *args)
-      super
+      @request_count += 1
+      @client.send(method_name, *args)
     rescue *@options[:exception_classes] => e
       disconnect_on_error!
-      tries ||= (@options[:retry_overrides][method_name.to_sym] || @retries) + 1
+      tries ||= (@options[:retry_overrides][method_name.to_sym] || @options[:retries]) + 1
       tries -= 1
-      tries > 0 ? retry : raise
-    end
-
-    def send_rpc(method_name, *args)
-      @request_count += 1
-      super
-    end
-
-    def disconnect_on_max!
-      @live_server_list.push(@current_server)
-      disconnect_on_error!
+      if tries > 0
+        retry
+      else
+        raise_or_default(e, method_name)
+      end
+    rescue Exception => e
+      raise_or_default(e, method_name)
     end
+  end
 
-    def disconnect_on_error!
-      super
-      @request_count = 0
+  def raise_or_default(e, method_name)
+    if @options[:raise]
+      raise_wrapped_error(e)
+    else
+      @options[:defaults][method_name.to_sym]
     end
-
   end
 
-  module TimingOutThriftClient
-    TIMINGOUT_DEFAULTS = {
-      :timeout => 1,
-      :timeout_overrides => {}
-    }.freeze
-
-    def initialize(client_class, servers, options = {})
-      super
-      @options = TIMINGOUT_DEFAULTS.merge(@options)
+  def raise_wrapped_error(e)
+    if @options[:wrapped_exception_classes].include?(e.class)
+      raise @client_class.const_get(e.class.to_s.split('::').last), e.message, e.backtrace
+    else
+      raise e
     end
+  end
 
-    private
-    def post_connect(method_name)
-      return unless has_timeouts?
-      @client.timeout = @options[:timeout_overrides][method_name.to_sym] || @options[:timeout]
-    end
+  def disconnect_on_max!
+    disconnect!
+  end
 
-    def has_timeouts?
-      @has_timeouts ||= has_timeouts!
-    end
+  def disconnect_on_error!
+    @current_server.mark_down!
+    disconnect!
+  end
 
-    def has_timeouts!
-      @options[:timeout_overrides].any? && transport_can_timeout?
-    end
+  def has_timeouts?
+    @has_timeouts ||= @options[:timeout_overrides].any? && transport_can_timeout?
+  end
 
-    def transport_can_timeout?
-      if (@options[:transport_wrapper] || @options[:transport]).method_defined?(:timeout=)
-        true
-      else
-        warn "ThriftClient: Timeout overrides have no effect with with transport type #{(@options[:transport_wrapper] || @options[:transport])}"
-        false
-      end
+  def transport_can_timeout?
+    if (@options[:transport_wrapper] || @options[:transport]).method_defined?(:timeout=)
+      true
+    else
+      warn "ThriftClient: Timeout overrides have no effect with with transport type #{(@options[:transport_wrapper] || @options[:transport])}"
+      false
     end
   end
 end

data/lib/thrift_client/event_machine.rb

@@ -27,8 +27,6 @@ module Thrift
       end
       Fiber.yield
 
-      # Use Thrift::TransportException so the RetryingThriftClient knows to try the next
-      # server instead of raising the error.
       raise Thrift::TransportException, "Unable to connect to #{@host}:#{@port}" unless @connection.connected?
       @connection
     end

data/lib/thrift_client.rb

@@ -1,11 +1,6 @@
-if ENV["ANCIENT_THRIFT"]
-  $LOAD_PATH.unshift("/Users/eweaver/p/twitter/rails/vendor/gems/thrift-751142/lib")
-  $LOAD_PATH.unshift("/Users/eweaver/p/twitter/rails/vendor/gems/thrift-751142/ext")
-  require 'thrift'
-else
-  require 'rubygems'
-  require 'thrift'
-end
+require 'rubygems'
+gem 'thrift', '~>0.2.0'
+require 'thrift'
 
 require 'rubygems'
 require 'thrift_client/thrift'
@@ -13,11 +8,7 @@ require 'thrift_client/connection'
 require 'thrift_client/abstract_thrift_client'
 
 class ThriftClient < AbstractThriftClient
-  # This error is for backwards compatibility only. If defined in
-  # RetryingThriftClient instead, causes the test suite will break.
   class NoServersAvailable < StandardError; end
-  include RetryingThriftClient
-  include TimingOutThriftClient
 
 =begin rdoc
 Create a new ThriftClient instance. Accepts an internal Thrift client class (such as CassandraRb::Client), a list of servers with ports, and optional parameters.
@@ -28,11 +19,10 @@ Valid optional parameters are:
 <tt>:protocol_extra_params</tt>:: An array of additional parameters to pass to the protocol initialization call. Defaults to <tt>[]</tt>.
 <tt>:transport</tt>:: Which Thrift transport to use. Defaults to <tt>Thrift::Socket</tt>.
 <tt>:transport_wrapper</tt>:: Which Thrift transport wrapper to use. Defaults to <tt>Thrift::FramedTransport</tt>.
-<tt>:randomize_server_list</tt>:: Whether to connect to the servers randomly, instead of in order. Defaults to <tt>true</tt>.
-<tt>:exception_classes</tt>:: Which exceptions to catch and retry when sending a request. Defaults to <tt>[IOError, Thrift::Exception, Thrift::ProtocolException, Thrift::ApplicationException, Thrift::TransportException, NoServersAvailable]</tt>
+<tt>:exception_classes</tt>:: Which exceptions to catch and retry when sending a request. Defaults to <tt>[IOError, Thrift::Exception, Thrift::ApplicationException, Thrift::TransportException, NoServersAvailable]</tt>
 <tt>:raise</tt>:: Whether to reraise errors if no responsive servers are found. Defaults to <tt>true</tt>.
-<tt>:retries</tt>:: How many times to retry a request. Defaults to the number of servers defined.
-<tt>:server_retry_period</tt>:: How many seconds to wait before trying to reconnect after marking all servers as down. Defaults to <tt>1</tt>. Set to <tt>nil</tt> to retry endlessly.
+<tt>:retries</tt>:: How many times to retry a request. Defaults to 0.
+<tt>:server_retry_period</tt>:: How many seconds to wait before trying to reconnect to a dead server. Defaults to <tt>1</tt>. Set to <tt>nil</tt> to disable.
 <tt>:server_max_requests</tt>:: How many requests to perform before moving on to the next server in the pool, regardless of error status. Defaults to <tt>nil</tt> (no limit).
 <tt>:timeout</tt>:: Specify the default timeout in seconds. Defaults to <tt>1</tt>.
 <tt>:timeout_overrides</tt>:: Specify additional timeouts on a per-method basis, in seconds. Only works with <tt>Thrift::BufferedTransport</tt>.

data/test/thrift_client_http_test.rb

@@ -42,13 +42,5 @@ class ThriftClientHTTPTest < Test::Unit::TestCase
       ThriftClient.new(Greeter::Client, "http://127.0.0.1:1463/greeter", @options).greeting("someone")
     end
   end
-  
-  def test_non_random_fall_through
-    @servers = ["http://127.0.0.1:1463/greeter", "http://127.0.0.1:1461/greeter", "http://127.0.0.1:1462/greeter"]
-    assert_nothing_raised do
-      @options.merge!({ :protocol => Thrift::BinaryProtocol, :transport => Thrift::HTTPClientTransport })
-      ThriftClient.new(Greeter::Client, @servers, @options.merge(:randomize_server_list => false)).greeting("someone")
-    end
-  end
 
 end

data/test/thrift_client_test.rb

@@ -32,33 +32,12 @@ class ThriftClientTest < Test::Unit::TestCase
     end
   end
 
-  def test_non_random_fall_through
-    assert_nothing_raised do
-      ThriftClient.new(Greeter::Client, @servers, @options.merge(:randomize_server_list => false)).greeting("someone")
-    end
-  end
-
   def test_dont_raise
     assert_nothing_raised do
       ThriftClient.new(Greeter::Client, @servers.first, @options.merge(:raise => false)).greeting("someone")
     end
   end
 
-  def test_handle_exception_is_called_once_when_retrying
-    client = ThriftClient.new(Greeter::Client, @servers.first, @options.merge(:raise => false, :retries => 1))
-    client.options[:exception_classes] += [ThriftClient::NoServersAvailable]
-    singleton_class = (class << client; self end)
-
-    times_called = 0
-    singleton_class.send :define_method, :handle_exception do |*args|
-      times_called += 1
-      raise IOError
-    end
-
-    assert_raises(IOError) { client.greeting('someone') }
-    assert_equal 1, times_called
-  end
-
   def test_retries_correct_number_of_times
     stub_server(@port) do |socket|
       opts = @options.merge(:timeout => @timeout, :retries => 4, :server_retry_period => nil)
@@ -73,7 +52,7 @@ class ThriftClientTest < Test::Unit::TestCase
         times_called += 1; super
       end
 
-      assert_raises(Thrift::TransportException) { client.greeting("someone") }
+      assert_raises(Greeter::Client::TransportException) { client.greeting("someone") }
       assert_equal opts[:retries] + 1, times_called
     end
   end
@@ -91,7 +70,7 @@ class ThriftClientTest < Test::Unit::TestCase
   def test_random_fall_through
     assert_nothing_raised do
       10.times do
-        client = ThriftClient.new(Greeter::Client, @servers, @options)
+        client = ThriftClient.new(Greeter::Client, @servers, @options.merge(:retries => 2))
         client.greeting("someone")
         client.disconnect!
       end
@@ -105,7 +84,7 @@ class ThriftClientTest < Test::Unit::TestCase
   end
 
   def test_no_servers_eventually_raise
-    client = ThriftClient.new(Greeter::Client, @servers[0,2], @options)
+    client = ThriftClient.new(Greeter::Client, @servers[0,2], @options.merge(:retries => 2))
     assert_raises(ThriftClient::NoServersAvailable) do
       client.greeting("someone")
       client.disconnect!
@@ -115,7 +94,7 @@ class ThriftClientTest < Test::Unit::TestCase
   def test_framed_transport_timeout
     stub_server(@port) do |socket|
       measurement = Benchmark.measure do
-        assert_raises(Thrift::TransportException) do
+        assert_raises(Greeter::Client::TransportException) do
           ThriftClient.new(Greeter::Client, "127.0.0.1:#{@port}",
             @options.merge(:timeout => @timeout)
           ).greeting("someone")
@@ -128,10 +107,11 @@ class ThriftClientTest < Test::Unit::TestCase
   def test_buffered_transport_timeout
     stub_server(@port) do |socket|
       measurement = Benchmark.measure do
-        assert_raises(Thrift::TransportException) do
-          ThriftClient.new(Greeter::Client, "127.0.0.1:#{@port}",
-            @options.merge(:timeout => @timeout, :transport_wrapper => Thrift::BufferedTransport)
-          ).greeting("someone")
+        client = ThriftClient.new(Greeter::Client, "127.0.0.1:#{@port}",
+          @options.merge(:timeout => @timeout, :transport_wrapper => Thrift::BufferedTransport)
+        )
+        assert_raises(Greeter::Client::TransportException) do
+          client.greeting("someone")
         end
       end
       assert((measurement.real > @timeout), "#{measurement.real} < #{@timeout}")
@@ -143,10 +123,11 @@ class ThriftClientTest < Test::Unit::TestCase
     log_timeout = @timeout * 4
     stub_server(@port) do |socket|
       measurement = Benchmark.measure do
-        assert_raises(Thrift::TransportException) do
-          ThriftClient.new(Greeter::Client, "127.0.0.1:#{@port}",
-            @options.merge(:timeout => @timeout, :timeout_overrides => {:greeting => log_timeout}, :transport_wrapper => Thrift::BufferedTransport)
-          ).greeting("someone")
+        client = ThriftClient.new(Greeter::Client, "127.0.0.1:#{@port}",
+          @options.merge(:timeout => @timeout, :timeout_overrides => {:greeting => log_timeout}, :transport_wrapper => Thrift::BufferedTransport)
+        )
+        assert_raises(Greeter::Client::TransportException) do
+          client.greeting("someone")
         end
       end
       assert((measurement.real > log_timeout), "#{measurement.real} < #{log_timeout}")
@@ -154,28 +135,28 @@ class ThriftClientTest < Test::Unit::TestCase
   end
 
   def test_retry_period
-    client = ThriftClient.new(Greeter::Client, @servers[0,2], @options.merge(:server_retry_period => 1))
+    client = ThriftClient.new(Greeter::Client, @servers[0,2], @options.merge(:server_retry_period => 1, :retries => 2))
     assert_raises(ThriftClient::NoServersAvailable) { client.greeting("someone") }
     sleep 1.1
     assert_raises(ThriftClient::NoServersAvailable) { client.greeting("someone") }
   end
 
   def test_client_with_retry_period_drops_servers
-    client = ThriftClient.new(Greeter::Client, @servers[0,2], @options.merge(:server_retry_period => 1))
+    client = ThriftClient.new(Greeter::Client, @servers[0,2], @options.merge(:server_retry_period => 1, :retries => 2))
     assert_raises(ThriftClient::NoServersAvailable) { client.greeting("someone") }
     sleep 1.1
     assert_raises(ThriftClient::NoServersAvailable) { client.greeting("someone") }
   end
 
   def test_oneway_method
-    client = ThriftClient.new(Greeter::Client, @servers, @options.merge(:server_max_requests => 2))
+    client = ThriftClient.new(Greeter::Client, @servers, @options.merge(:server_max_requests => 2, :retries => 2))
     assert_nothing_raised do
       response = client.yo("dude")
     end
   end
 
   def test_server_max_requests_with_downed_servers
-    client = ThriftClient.new(Greeter::Client, @servers, @options.merge(:server_max_requests => 2))
+    client = ThriftClient.new(Greeter::Client, @servers, @options.merge(:server_max_requests => 2, :retries => 2))
     client.greeting("someone")
     internal_client = client.client
     client.greeting("someone")

data/thrift_client.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{thrift_client}
-  s.version = "0.4.7"
+  s.version = "0.5.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0.8") if s.respond_to? :required_rubygems_version=
   s.authors = ["Evan Weaver, Ryan King, Jeff Hodges"]

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: thrift_client
 version: !ruby/object:Gem::Version 
-  hash: 1
+  hash: 11
   prerelease: false
   segments: 
   - 0
-  - 4
-  - 7
-  version: 0.4.7
+  - 5
+  - 0
+  version: 0.5.0
 platform: ruby
 authors: 
 - Evan Weaver, Ryan King, Jeff Hodges