elasticsearch-transport 1.0.14 → 1.0.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a1c41acb9864d25738a29b7ced81856e5eea8660
-  data.tar.gz: 1675e534fedd834b107e51c620767f11e1b9327f
+  metadata.gz: c8912230f142b53ab174f9cb27d98c3faa220517
+  data.tar.gz: d561830a59e77b6daf839a4b7fef09394506ea98
 SHA512:
-  metadata.gz: 2473aa1f15584614fcf9c96e75030bd37979817ba4ddcce78134781b275deedd35cf1a62a46af649c704d2d84a247751fe680be1c111de8ac8d25e6c33316817
-  data.tar.gz: 18aa296103d2c29b5ae11a3258f7e9c5f97c6787ad061ad4d0713a9e7fb916d920fb0c92604065bbbedf6fd7e15f75d2a253238a7cb26de141e7c48f0727f616
+  metadata.gz: 6a41d9e20a098bb9731e7029daf96996c92e898b982fef262ac290cd680dde7a5d99d8d6134aaca5c9a1df31dc525aeee603e0881d80ae972315b7524579e7e6
+  data.tar.gz: 83667f70dc3c27c925f63ec0def5400f292f60bfe4b9b83629c74a243adb8a0dd23ecd45dd247c9530da06bb7429411cddc10355b00c40a1e64e6250c21e9062

data/README.md

@@ -271,24 +271,6 @@ To use a specific adapter for _Faraday_, pass it as the `adapter` argument:
     client.transport.connections.first.connection.builder.handlers
     # => [Faraday::Adapter::NetHttpPersistent]
 
-To configure the _Faraday_ instance, pass a configuration block to the transport constructor:
-
-    require 'typhoeus'
-    require 'typhoeus/adapters/faraday'
-
-    transport_configuration = lambda do |f|
-      f.response :logger
-      f.adapter  :typhoeus
-    end
-
-    transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new \
-      hosts: [ { host: 'localhost', port: '9200' } ],
-      &transport_configuration
-
-    # Pass the transport to the client
-    #
-    client = Elasticsearch::Client.new transport: transport
-
 To pass options to the
 [`Faraday::Connection`](https://github.com/lostisland/faraday/blob/master/lib/faraday/connection.rb)
 constructor, use the `transport_options` key:
@@ -300,23 +282,46 @@ constructor, use the `transport_options` key:
       ssl:     { verify: false }
     }
 
-You can also use a bundled [_Curb_](https://rubygems.org/gems/curb) based transport implementation:
+To configure the _Faraday_ instance directly, use a block:
 
-    require 'curb'
-    require 'elasticsearch/transport/transport/http/curb'
+    require 'typhoeus'
+    require 'typhoeus/adapters/faraday'
 
-    client = Elasticsearch::Client.new transport_class: Elasticsearch::Transport::Transport::HTTP::Curb
+    client = Elasticsearch::Client.new(host: 'localhost', port: '9200') do |f|
+      f.response :logger
+      f.adapter  :typhoeus
+    end
 
-    client.transport.connections.first.connection
-    # => #<Curl::Easy http://localhost:9200/>
+You can use any standard Faraday middleware and plugins in the configuration block,
+for example sign the requests for the [AWS Elasticsearch service](https://aws.amazon.com/elasticsearch-service/):
 
-It's possible to customize the _Curb_ instance by passing a block to the constructor as well
-(in this case, as an inline block):
+    require 'patron'
+    require 'faraday_middleware/aws_signers_v4'
 
-    transport = Elasticsearch::Transport::Transport::HTTP::Curb.new \
+    client = Elasticsearch::Client.new url: 'https://search-my-cluster-abc123....es.amazonaws.com' do |f|
+      f.request :aws_signers_v4,
+                credentials: Aws::Credentials.new(ENV['AWS_ACCESS_KEY'], ENV['AWS_SECRET_ACCESS_KEY']),
+                service_name: 'es',
+                region: 'us-east-1'
+    end
+
+You can also initialize the transport class yourself, and pass it to the client constructor
+as the `transport` argument:
+
+    require 'typhoeus'
+    require 'typhoeus/adapters/faraday'
+
+    transport_configuration = lambda do |f|
+      f.response :logger
+      f.adapter  :typhoeus
+    end
+
+    transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new \
       hosts: [ { host: 'localhost', port: '9200' } ],
-      & lambda { |c| c.verbose = true }
+      &transport_configuration
 
+    # Pass the transport to the client
+    #
     client = Elasticsearch::Client.new transport: transport
 
 Instead of passing the transport to the constructor, you can inject it at run time:
@@ -345,6 +350,25 @@ Instead of passing the transport to the constructor, you can inject it at run ti
     #
     client.transport = faraday_client
 
+You can also use a bundled [_Curb_](https://rubygems.org/gems/curb) based transport implementation:
+
+    require 'curb'
+    require 'elasticsearch/transport/transport/http/curb'
+
+    client = Elasticsearch::Client.new transport_class: Elasticsearch::Transport::Transport::HTTP::Curb
+
+    client.transport.connections.first.connection
+    # => #<Curl::Easy http://localhost:9200/>
+
+It's possible to customize the _Curb_ instance by passing a block to the constructor as well
+(in this case, as an inline block):
+
+    transport = Elasticsearch::Transport::Transport::HTTP::Curb.new \
+      hosts: [ { host: 'localhost', port: '9200' } ],
+      & lambda { |c| c.verbose = true }
+
+    client = Elasticsearch::Client.new transport: transport
+
 You can write your own transport implementation easily, by including the
 {Elasticsearch::Transport::Transport::Base} module, implementing the required contract,
 and passing it to the client as the `transport_class` parameter -- or injecting it directly.

data/elasticsearch-transport.gemspec

@@ -46,6 +46,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency "curb"   unless defined? JRUBY_VERSION
   s.add_development_dependency "patron" unless defined? JRUBY_VERSION
   s.add_development_dependency "typhoeus", '~> 0.6'
+  s.add_development_dependency "net-http-persistent"
   s.add_development_dependency "manticore", '~> 0.3.5' if defined? JRUBY_VERSION
   s.add_development_dependency "hashie"
 

data/lib/elasticsearch/transport.rb

@@ -22,8 +22,8 @@ module Elasticsearch
 
     # A convenience wrapper for {::Elasticsearch::Transport::Client#initialize}.
     #
-    def new(arguments={})
-      Elasticsearch::Transport::Client.new(arguments)
+    def new(arguments={}, &block)
+      Elasticsearch::Transport::Client.new(arguments, &block)
     end
     extend self
   end

data/lib/elasticsearch/transport/client.rb

@@ -56,6 +56,7 @@ module Elasticsearch
       #
       # @option arguments [Boolean,Number] :retry_on_failure   Retry X times when request fails before raising and
       #                                                        exception (false by default)
+      # @option arguments Array<Number> :retry_on_status Retry when specific status codes are returned
       #
       # @option arguments [Boolean] :reload_on_failure Reload connections after failure (false by default)
       #
@@ -79,7 +80,9 @@ module Elasticsearch
       # @option arguments [String] :send_get_body_as Specify the HTTP method to use for GET requests with a body.
       #                                              (Default: GET)
       #
-      def initialize(arguments={})
+      # @yield [faraday] Access and configure the `Faraday::Connection` instance directly with a block
+      #
+      def initialize(arguments={}, &block)
         hosts = arguments[:hosts] || \
                 arguments[:host]  || \
                 arguments[:url]   || \
@@ -103,7 +106,10 @@ module Elasticsearch
         @transport       = arguments[:transport] || begin
           if transport_class == Transport::HTTP::Faraday
             transport_class.new(:hosts => __extract_hosts(hosts, arguments), :options => arguments) do |faraday|
-              faraday.adapter(arguments[:adapter] || __auto_detect_adapter)
+              block.call faraday if block
+              unless (h = faraday.builder.handlers.last) && h.name.start_with?("Faraday::Adapter")
+                faraday.adapter(arguments[:adapter] || __auto_detect_adapter)
+              end
             end
           else
             transport_class.new(:hosts => __extract_hosts(hosts, arguments), :options => arguments)

data/lib/elasticsearch/transport/transport/base.rb

@@ -47,6 +47,7 @@ module Elasticsearch
           @reload_after    = options[:reload_connections].is_a?(Fixnum) ? options[:reload_connections] : DEFAULT_RELOAD_AFTER
           @resurrect_after = options[:resurrect_after] || DEFAULT_RESURRECT_AFTER
           @max_retries     = options[:retry_on_failure].is_a?(Fixnum)   ? options[:retry_on_failure]   : DEFAULT_MAX_RETRIES
+          @retry_on_status = Array(options[:retry_on_status]).map { |d| d.to_i }
         end
 
         # Returns a connection from the connection pool by delegating to {Connections::Collection#get_connection}.
@@ -95,9 +96,18 @@ module Elasticsearch
         def __rebuild_connections(arguments={})
           @hosts       = arguments[:hosts]    || []
           @options     = arguments[:options]  || {}
+          __close_connections
           @connections = __build_connections
         end
 
+        # Closes the connections collection.
+        #
+        # @api private
+        #
+        def __close_connections
+          # to be implemented by specific transports
+        end
+
         # Log request and response information.
         #
         # @api private
@@ -192,6 +202,23 @@ module Elasticsearch
 
             connection.healthy! if connection.failures > 0
 
+            # Raise an exception so we can catch it for `retry_on_status`
+            __raise_transport_error(response) if response.status.to_i >= 300 && @retry_on_status.include?(response.status.to_i)
+
+          rescue Elasticsearch::Transport::Transport::ServerError => e
+            if @retry_on_status.include?(response.status)
+              logger.warn "[#{e.class}] Attempt #{tries} to get response from #{url}" if logger
+              logger.debug "[#{e.class}] Attempt #{tries} to get response from #{url}" if logger
+              if tries <= max_retries
+                retry
+              else
+                logger.fatal "[#{e.class}] Cannot get response from #{url} after #{tries} tries" if logger
+                raise e
+              end
+            else
+              raise e
+            end
+
           rescue *host_unreachable_exceptions => e
             logger.error "[#{e.class}] #{e.message} #{connection.host.inspect}" if logger
 
@@ -217,7 +244,8 @@ module Elasticsearch
           rescue Exception => e
             logger.fatal "[#{e.class}] #{e.message} (#{connection.host.inspect if connection})" if logger
             raise e
-          end
+
+          end #/begin
 
           duration = Time.now-start if logger || tracer
 

data/lib/elasticsearch/transport/transport/http/manticore.rb

@@ -105,6 +105,16 @@ module Elasticsearch
               :selector => options[:selector]
           end
 
+          # Closes all connections by marking them as dead
+          # and closing the underlying HttpClient instances
+          #
+          # @return [Connections::Collection]
+          #
+          def __close_connections
+            @connections.each {|c| c.dead! }
+            @connections.all.each {|c| c.connection.close }
+          end
+
           # Returns an array of implementation specific connection errors.
           #
           # @return [Array]

data/lib/elasticsearch/transport/version.rb

@@ -1,5 +1,5 @@
 module Elasticsearch
   module Transport
-    VERSION = "1.0.14"
+    VERSION = "1.0.15"
   end
 end

data/test/integration/client_test.rb

@@ -11,8 +11,9 @@ class Elasticsearch::Transport::ClientIntegrationTest < Elasticsearch::Test::Int
 
   context "Elasticsearch client" do
     teardown do
-      begin; Object.send(:remove_const, :Typhoeus); rescue NameError; end
-      begin; Object.send(:remove_const, :Patron); rescue NameError; end
+      begin; Object.send(:remove_const, :Typhoeus);                rescue NameError; end
+      begin; Object.send(:remove_const, :Patron);                  rescue NameError; end
+      begin; Net::HTTP.send(:remove_const, :Persistent); rescue NameError; end
     end
 
     setup do
@@ -63,6 +64,20 @@ class Elasticsearch::Transport::ClientIntegrationTest < Elasticsearch::Test::Int
       assert_match /---\ncluster_name:/, response.body.to_s
     end
 
+    should "pass options to the Faraday::Connection with a block" do
+      @client = Elasticsearch::Client.new(
+        host: "localhost:#{@port}",
+        logger: (ENV['QUIET'] ? nil : @logger)
+      ) do |client|
+        client.headers['Content-Type'] = 'application/yaml'
+      end
+
+      response = @client.perform_request 'GET', '_cluster/health'
+
+      assert response.body.start_with?("---\n"), "Response body should be YAML: #{response.body.inspect}"
+      assert_equal 'application/yaml', response.headers['content-type']
+    end
+
     context "with round robin selector" do
       setup do
         @client = Elasticsearch::Client.new \
@@ -135,11 +150,46 @@ class Elasticsearch::Transport::ClientIntegrationTest < Elasticsearch::Test::Int
       end
     end
 
+    context "with retrying on status" do
+      should "retry when the status does match" do
+        @client = Elasticsearch::Client.new \
+                  hosts: ["localhost:#{@port}"],
+                  logger: (ENV['QUIET'] ? nil : @logger),
+                  retry_on_status: 400
+
+        @client.transport.logger
+          .expects(:debug)
+          .with( regexp_matches(/Attempt \d to get response/) )
+          .times(4)
+
+        assert_raise Elasticsearch::Transport::Transport::Errors::BadRequest do
+          @client.perform_request 'GET', '_foobar'
+        end
+      end
+    end
+
     context "with Faraday adapters" do
+      should "set the adapter with a block" do
+        require 'net/http/persistent'
+
+        client = Elasticsearch::Transport::Client.new url: "localhost:#{@port}" do |f|
+          f.adapter :net_http_persistent
+        end
+
+        assert_equal 'Faraday::Adapter::NetHttpPersistent',
+                     client.transport.connections.first.connection.builder.handlers.first.name
+
+        response = @client.perform_request 'GET', '_cluster/health'
+        assert_equal 200, response.status
+      end
+
       should "automatically use the Patron client when loaded" do
         require 'patron'
         client = Elasticsearch::Transport::Client.new host: "localhost:#{@port}"
 
+        assert_equal 'Faraday::Adapter::Patron',
+                      client.transport.connections.first.connection.builder.handlers.first.name
+
         response = @client.perform_request 'GET', '_cluster/health'
         assert_equal 200, response.status
       end unless JRUBY

data/test/unit/client_test.rb

@@ -270,5 +270,30 @@ class Elasticsearch::Transport::ClientTest < Test::Unit::TestCase
       end unless JRUBY
     end
 
+    context "configuring Faraday" do
+      setup do
+        Elasticsearch::Transport::Client::DEFAULT_TRANSPORT_CLASS.any_instance.unstub(:__build_connections)
+        begin; Object.send(:remove_const, :Typhoeus); rescue NameError; end
+      end
+
+      should "apply faraday adapter" do
+        c = Elasticsearch::Transport::Client.new do |faraday|
+          faraday.adapter :typhoeus
+        end
+        handlers = c.transport.connections.all.first.connection.builder.handlers
+
+        assert_includes handlers, Faraday::Adapter::Typhoeus
+      end
+
+      should "apply faraday response logger" do
+        c = Elasticsearch::Transport::Client.new do |faraday|
+          faraday.response :logger
+        end
+        handlers = c.transport.connections.all.first.connection.builder.handlers
+
+        assert_includes handlers, Faraday::Response::Logger
+      end
+    end
+
   end
 end

data/test/unit/transport_base_test.rb

@@ -296,6 +296,53 @@ class Elasticsearch::Transport::Transport::BaseTest < Test::Unit::TestCase
     end
   end unless RUBY_1_8
 
+  context "performing a request with retry on status" do
+    setup do
+      DummyTransportPerformer.any_instance.stubs(:connections).returns(stub :get_connection => stub_everything(:failures => 1))
+
+      logger = Logger.new(STDERR)
+      logger.level = Logger::DEBUG
+      DummyTransportPerformer.any_instance.stubs(:logger).returns(logger)
+      @block = Proc.new { |c, u| puts "ERROR" }
+    end
+
+    should "not retry when the status code does not match" do
+      @transport = DummyTransportPerformer.new :options => { :retry_on_status => 500 }
+      assert_equal [500], @transport.instance_variable_get(:@retry_on_status)
+
+      @block.expects(:call).
+             returns(Elasticsearch::Transport::Transport::Response.new 400, 'Bad Request').
+             times(1)
+
+      @transport.logger.
+          expects(:warn).
+          with( regexp_matches(/Attempt \d to get response/) ).
+          never
+
+      assert_raise Elasticsearch::Transport::Transport::Errors::BadRequest do
+        @transport.perform_request('GET', '/', &@block)
+      end
+    end
+
+    should "retry when the status code does match" do
+      @transport = DummyTransportPerformer.new :options => { :retry_on_status => 500 }
+      assert_equal [500], @transport.instance_variable_get(:@retry_on_status)
+
+      @block.expects(:call).
+             returns(Elasticsearch::Transport::Transport::Response.new 500, 'Internal Error').
+             times(4)
+
+      @transport.logger.
+          expects(:warn).
+          with( regexp_matches(/Attempt \d to get response/) ).
+          times(4)
+
+      assert_raise Elasticsearch::Transport::Transport::Errors::InternalServerError do
+        @transport.perform_request('GET', '/', &@block)
+      end
+    end
+  end  unless RUBY_1_8
+
   context "logging" do
     setup do
       @transport = DummyTransportPerformer.new :options => { :logger => Logger.new('/dev/null') }
@@ -426,6 +473,11 @@ class Elasticsearch::Transport::Transport::BaseTest < Test::Unit::TestCase
       @transport = DummyTransport.new
     end
 
+    should "close connections" do
+      @transport.expects(:__close_connections)
+      @transport.__rebuild_connections :hosts => ['foo']
+    end
+
     should "should replace the connections" do
       assert_equal [], @transport.connections
       @transport.__rebuild_connections :hosts => ['foo', 'bar']

data/test/unit/transport_manticore_test.rb

@@ -26,6 +26,19 @@ else
         assert_instance_of ::Manticore::Client,   @transport.connections.first.connection
       end
 
+      should "implement __close_connections" do
+        assert_equal 1, @transport.connections.size
+        @transport.__close_connections
+        assert_equal 0, @transport.connections.size
+      end
+
+      should "prevent requests after __close_connections" do
+        @transport.__close_connections
+        assert_raises ::Manticore::ClientStoppedException do
+          @transport.perform_request 'GET', '/'
+        end
+      end
+
       should "perform the request" do
         @transport.connections.first.connection.expects(:get).returns(stub_everything)
         @transport.perform_request 'GET', '/'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elasticsearch-transport
 version: !ruby/object:Gem::Version
-  version: 1.0.14
+  version: 1.0.15
 platform: ruby
 authors:
 - Karel Minarik
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-14 00:00:00.000000000 Z
+date: 2015-12-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -220,6 +220,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: net-http-persistent
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: hashie
   requirement: !ruby/object:Gem::Requirement