elasticsearch-transport 7.1.0 → 7.8.0

data/lib/elasticsearch/transport/redacted.rb

@@ -6,7 +6,7 @@
 # not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#	http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an

data/lib/elasticsearch/transport/transport/base.rb

@@ -6,7 +6,7 @@
 # not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#	http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
@@ -35,7 +35,7 @@ module Elasticsearch
         attr_reader   :hosts, :options, :connections, :counter, :last_request_at, :protocol
         attr_accessor :serializer, :sniffer, :logger, :tracer,
                       :reload_connections, :reload_after,
-                      :resurrect_after, :max_retries
+                      :resurrect_after
 
         # Creates a new transport object
         #
@@ -56,6 +56,7 @@ module Elasticsearch
           @options[:retry_on_status] ||= []
 
           @block       = block
+          @compression = !!@options[:compression]
           @connections = __build_connections
 
           @serializer  = options[:serializer] || ( options[:serializer_class] ? options[:serializer_class].new(self) : DEFAULT_SERIALIZER_CLASS.new(self) )
@@ -71,7 +72,6 @@ module Elasticsearch
           @reload_connections = options[:reload_connections]
           @reload_after    = options[:reload_connections].is_a?(Integer) ? options[:reload_connections] : DEFAULT_RELOAD_AFTER
           @resurrect_after = options[:resurrect_after] || DEFAULT_RESURRECT_AFTER
-          @max_retries     = options[:retry_on_failure].is_a?(Integer)   ? options[:retry_on_failure]   : DEFAULT_MAX_RETRIES
           @retry_on_status = Array(options[:retry_on_status]).map { |d| d.to_i }
         end
 
@@ -202,7 +202,7 @@ module Elasticsearch
               ( params.empty? ? '' : "&#{::Faraday::Utils::ParamsHash[params].to_query}" )
           trace_body = body ? " -d '#{__convert_to_json(body, :pretty => true)}'" : ''
           trace_command = "curl -X #{method.to_s.upcase}"
-          trace_command += " -H '#{headers.inject('') { |memo,item| memo << item[0] + ': ' + item[1] }}'" if headers && !headers.empty?
+          trace_command += " -H '#{headers.collect { |k,v| "#{k}: #{v}" }.join(", ")}'" if headers && !headers.empty?
           trace_command += " '#{trace_url}'#{trace_body}\n"
           tracer.info trace_command
           tracer.debug "# #{Time.now.iso8601} [#{response.status}] (#{format('%.3f', duration)}s)\n#"
@@ -234,7 +234,8 @@ module Elasticsearch
         def __full_url(host)
           url  = "#{host[:protocol]}://"
           url += "#{CGI.escape(host[:user])}:#{CGI.escape(host[:password])}@" if host[:user]
-          url += "#{host[:host]}:#{host[:port]}"
+          url += "#{host[:host]}"
+          url += ":#{host[:port]}" if host[:port]
           url += "#{host[:path]}" if host[:path]
           url
         end
@@ -257,10 +258,17 @@ module Elasticsearch
         # @raise  [ServerError]   If request failed on server
         # @raise  [Error]         If no connection is available
         #
-        def perform_request(method, path, params={}, body=nil, headers=nil, &block)
+        def perform_request(method, path, params={}, body=nil, headers=nil, opts={}, &block)
           raise NoMethodError, "Implement this method in your transport class" unless block_given?
           start = Time.now
           tries = 0
+          reload_on_failure = opts.fetch(:reload_on_failure, @options[:reload_on_failure])
+
+          max_retries = if opts.key?(:retry_on_failure)
+            opts[:retry_on_failure] === true ? DEFAULT_MAX_RETRIES : opts[:retry_on_failure]
+          elsif options.key?(:retry_on_failure)
+            options[:retry_on_failure] === true ? DEFAULT_MAX_RETRIES : options[:retry_on_failure]
+          end
 
           params = params.clone
 
@@ -286,7 +294,7 @@ module Elasticsearch
           rescue Elasticsearch::Transport::Transport::ServerError => e
             if response && @retry_on_status.include?(response.status)
               log_warn "[#{e.class}] Attempt #{tries} to get response from #{url}"
-              if tries <= max_retries
+              if tries <= (max_retries || DEFAULT_MAX_RETRIES)
                 retry
               else
                 log_fatal "[#{e.class}] Cannot get response from #{url} after #{tries} tries"
@@ -301,12 +309,12 @@ module Elasticsearch
 
             connection.dead!
 
-            if @options[:reload_on_failure] and tries < connections.all.size
+            if reload_on_failure and tries < connections.all.size
               log_warn "[#{e.class}] Reloading connections (attempt #{tries} of #{connections.all.size})"
               reload_connections! and retry
             end
 
-            if @options[:retry_on_failure]
+            if max_retries
               log_warn "[#{e.class}] Attempt #{tries} connecting to #{connection.host.inspect}"
               if tries <= max_retries
                 retry
@@ -328,7 +336,7 @@ module Elasticsearch
 
           if response.status.to_i >= 300
             __log_response    method, path, params, body, url, response, nil, 'N/A', duration
-            __trace  method, path, params, headers, body, url, response, nil, 'N/A', duration if tracer
+            __trace  method, path, params, connection.connection.headers, body, url, response, nil, 'N/A', duration if tracer
 
             # Log the failure only when `ignore` doesn't match the response status
             unless ignore.include?(response.status.to_i)
@@ -345,7 +353,9 @@ module Elasticsearch
             __log_response   method, path, params, body, url, response, json, took, duration
           end
 
-          __trace method, path, params, headers, body, url, response, json, took, duration if tracer
+          __trace  method, path, params, connection.connection.headers, body, url, response, nil, 'N/A', duration if tracer
+
+          warnings(response.headers['warning']) if response.headers&.[]('warning')
 
           Response.new response.status, json || response.body, response.headers
         ensure
@@ -360,7 +370,71 @@ module Elasticsearch
         def host_unreachable_exceptions
           [Errno::ECONNREFUSED]
         end
+
+        private
+
+        USER_AGENT_STR = 'User-Agent'.freeze
+        USER_AGENT_REGEX = /user\-?\_?agent/
+        CONTENT_TYPE_STR = 'Content-Type'.freeze
+        CONTENT_TYPE_REGEX = /content\-?\_?type/
+        DEFAULT_CONTENT_TYPE = 'application/json'.freeze
+        GZIP = 'gzip'.freeze
+        ACCEPT_ENCODING = 'Accept-Encoding'.freeze
+        GZIP_FIRST_TWO_BYTES = '1f8b'.freeze
+        HEX_STRING_DIRECTIVE = 'H*'.freeze
+        RUBY_ENCODING = '1.9'.respond_to?(:force_encoding)
+
+        def decompress_response(body)
+          return body unless use_compression?
+          return body unless gzipped?(body)
+
+          io = StringIO.new(body)
+          gzip_reader = if RUBY_ENCODING
+                          Zlib::GzipReader.new(io, :encoding => 'ASCII-8BIT')
+                        else
+                          Zlib::GzipReader.new(io)
+                        end
+          gzip_reader.read
+        end
+
+        def gzipped?(body)
+          body[0..1].unpack(HEX_STRING_DIRECTIVE)[0] == GZIP_FIRST_TWO_BYTES
+        end
+
+        def use_compression?
+          @compression
+        end
+
+        def apply_headers(client, options)
+          headers = options[:headers] || {}
+          headers[CONTENT_TYPE_STR] = find_value(headers, CONTENT_TYPE_REGEX) || DEFAULT_CONTENT_TYPE
+          headers[USER_AGENT_STR] = find_value(headers, USER_AGENT_REGEX) || user_agent_header(client)
+          client.headers[ACCEPT_ENCODING] = GZIP if use_compression?
+          client.headers.merge!(headers)
+        end
+
+        def find_value(hash, regex)
+          key_value = hash.find { |k,v| k.to_s.downcase =~ regex }
+          if key_value
+            hash.delete(key_value[0])
+            key_value[1]
+          end
+        end
+
+        def user_agent_header(client)
+          @user_agent ||= begin
+            meta = ["RUBY_VERSION: #{RUBY_VERSION}"]
+            if RbConfig::CONFIG && RbConfig::CONFIG['host_os']
+              meta << "#{RbConfig::CONFIG['host_os'].split('_').first[/[a-z]+/i].downcase} #{RbConfig::CONFIG['target_cpu']}"
+            end
+            "elasticsearch-ruby/#{VERSION} (#{meta.join('; ')})"
+          end
+        end
+
+        def warnings(warning)
+          warn("warning: #{warning}")
+        end
       end
     end
   end
-end
+end

data/lib/elasticsearch/transport/transport/connections/collection.rb

@@ -6,7 +6,7 @@
 # not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#	http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an

data/lib/elasticsearch/transport/transport/connections/connection.rb

@@ -6,7 +6,7 @@
 # not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#	http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an

data/lib/elasticsearch/transport/transport/connections/selector.rb

@@ -6,7 +6,7 @@
 # not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#	http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
@@ -61,18 +61,30 @@ module Elasticsearch
           class RoundRobin
             include Base
 
+            # @option arguments [Connections::Collection] :connections Collection with connections.
+            #
+            def initialize(arguments = {})
+              super
+              @mutex = Mutex.new
+              @current = nil
+            end
+
             # Returns the next connection from the collection, rotating them in round-robin fashion.
             #
             # @return [Connections::Connection]
             #
             def select(options={})
-              # On Ruby 1.9, Array#rotate could be used instead
-              @current = !defined?(@current) || @current.nil? ? 0 : @current+1
-              @current = 0 if @current >= connections.size
-              connections[@current]
+              @mutex.synchronize do
+                conns = connections
+                if @current && (@current < conns.size-1)
+                  @current += 1
+                else
+                  @current = 0
+                end
+                conns[@current]
+              end
             end
           end
-
         end
       end
     end

data/lib/elasticsearch/transport/transport/errors.rb

@@ -6,7 +6,7 @@
 # not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#	http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an

data/lib/elasticsearch/transport/transport/http/curb.rb

@@ -6,7 +6,7 @@
 # not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#	http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
@@ -32,7 +32,7 @@ module Elasticsearch
           # @return [Response]
           # @see    Transport::Base#perform_request
           #
-          def perform_request(method, path, params={}, body=nil, headers=nil)
+          def perform_request(method, path, params={}, body=nil, headers=nil, opts={})
             super do |connection, url|
               connection.connection.url = connection.full_url(path, params)
 
@@ -43,7 +43,15 @@ module Elasticsearch
                   connection.connection.set :nobody, false
 
                   connection.connection.put_data = __convert_to_json(body) if body
-                  connection.connection.headers = headers if headers
+
+                  if headers
+                    if connection.connection.headers
+                      connection.connection.headers.merge!(headers)
+                    else
+                      connection.connection.headers = headers
+                    end
+                  end
+
                 else raise ArgumentError, "Unsupported HTTP method: #{method}"
               end
 
@@ -53,7 +61,7 @@ module Elasticsearch
               response_headers['content-type'] = 'application/json' if connection.connection.header_str =~ /\/json/
 
               Response.new connection.connection.response_code,
-                           connection.connection.body_str,
+                           decompress_response(connection.connection.body_str),
                            response_headers
             end
           end
@@ -65,10 +73,7 @@ module Elasticsearch
           def __build_connection(host, options={}, block=nil)
             client = ::Curl::Easy.new
 
-            headers = options[:headers] || {}
-            headers.update('User-Agent' => "Curb #{Curl::CURB_VERSION}")
-
-            client.headers = headers
+            apply_headers(client, options)
             client.url     = __full_url(host)
 
             if host[:user]
@@ -96,8 +101,20 @@ module Elasticsearch
               ::Curl::Err::TimeoutError
             ]
           end
-        end
 
+          private
+
+          def user_agent_header(client)
+            @user_agent ||= begin
+              meta = ["RUBY_VERSION: #{RUBY_VERSION}"]
+              if RbConfig::CONFIG && RbConfig::CONFIG['host_os']
+                meta << "#{RbConfig::CONFIG['host_os'].split('_').first[/[a-z]+/i].downcase} #{RbConfig::CONFIG['target_cpu']}"
+              end
+              meta << "Curb #{Curl::CURB_VERSION}"
+              "elasticsearch-ruby/#{VERSION} (#{meta.join('; ')})"
+            end
+          end
+        end
       end
     end
   end

data/lib/elasticsearch/transport/transport/http/faraday.rb

@@ -6,7 +6,7 @@
 # not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#	http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
@@ -33,7 +33,7 @@ module Elasticsearch
           # @return [Response]
           # @see    Transport::Base#perform_request
           #
-          def perform_request(method, path, params={}, body=nil, headers=nil)
+          def perform_request(method, path, params={}, body=nil, headers=nil, opts={})
             super do |connection, url|
               headers = headers || connection.connection.headers
 
@@ -42,7 +42,7 @@ module Elasticsearch
                                                            ( body ? __convert_to_json(body) : nil ),
                                                            headers)
 
-              Response.new response.status, response.body, response.headers
+              Response.new response.status, decompress_response(response.body), response.headers
             end
           end
 
@@ -52,6 +52,7 @@ module Elasticsearch
           #
           def __build_connection(host, options={}, block=nil)
             client = ::Faraday.new(__full_url(host), options, &block)
+            apply_headers(client, options)
             Connections::Connection.new :host => host, :connection => client
           end
 
@@ -60,7 +61,20 @@ module Elasticsearch
           # @return [Array]
           #
           def host_unreachable_exceptions
-            [::Faraday::Error::ConnectionFailed, ::Faraday::Error::TimeoutError]
+            [::Faraday::ConnectionFailed, ::Faraday::TimeoutError]
+          end
+
+          private
+
+          def user_agent_header(client)
+            @user_agent ||= begin
+              meta = ["RUBY_VERSION: #{RUBY_VERSION}"]
+              if RbConfig::CONFIG && RbConfig::CONFIG['host_os']
+                meta << "#{RbConfig::CONFIG['host_os'].split('_').first[/[a-z]+/i].downcase} #{RbConfig::CONFIG['target_cpu']}"
+              end
+              meta << "#{client.headers[USER_AGENT_STR]}"
+              "elasticsearch-ruby/#{VERSION} (#{meta.join('; ')})"
+            end
           end
         end
       end

data/lib/elasticsearch/transport/transport/http/manticore.rb

@@ -6,7 +6,7 @@
 # not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#	http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
@@ -80,7 +80,7 @@ module Elasticsearch
           # @return [Response]
           # @see    Transport::Base#perform_request
           #
-          def perform_request(method, path, params={}, body=nil, headers=nil)
+          def perform_request(method, path, params={}, body=nil, headers=nil, opts={})
             super do |connection, url|
               params[:body] = __convert_to_json(body) if body
               params[:headers] = headers if headers
@@ -110,14 +110,8 @@ module Elasticsearch
           #
           def __build_connections
             @request_options = {}
-
-            if options[:transport_options] && options[:transport_options][:headers]
-              @request_options[:headers] = options[:transport_options][:headers]
-            end
-
-            if options.key?(:headers)
-              @request_options[:headers] = options[:headers]
-            end
+            apply_headers(@request_options, options[:transport_options])
+            apply_headers(@request_options, options)
 
             Connections::Collection.new \
               :connections => hosts.map { |host|
@@ -157,6 +151,27 @@ module Elasticsearch
               ::Manticore::ResolutionFailure
             ]
           end
+
+          private
+
+          def apply_headers(request_options, options)
+            headers = (options && options[:headers]) || {}
+            headers[CONTENT_TYPE_STR] = find_value(headers, CONTENT_TYPE_REGEX) || DEFAULT_CONTENT_TYPE
+            headers[USER_AGENT_STR] = find_value(headers, USER_AGENT_REGEX) || user_agent_header
+            headers[ACCEPT_ENCODING] = GZIP if use_compression?
+            request_options.merge!(headers: headers)
+          end
+
+          def user_agent_header
+            @user_agent ||= begin
+              meta = ["RUBY_VERSION: #{JRUBY_VERSION}"]
+              if RbConfig::CONFIG && RbConfig::CONFIG['host_os']
+                meta << "#{RbConfig::CONFIG['host_os'].split('_').first[/[a-z]+/i].downcase} #{RbConfig::CONFIG['target_cpu']}"
+              end
+              meta << "Manticore #{::Manticore::VERSION}"
+              "elasticsearch-ruby/#{VERSION} (#{meta.join('; ')})"
+            end
+          end
         end
       end
     end

data/lib/elasticsearch/transport/transport/loggable.rb

@@ -6,7 +6,7 @@
 # not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#	http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an

data/lib/elasticsearch/transport/transport/response.rb

@@ -6,7 +6,7 @@
 # not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#	http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
@@ -18,7 +18,6 @@
 module Elasticsearch
   module Transport
     module Transport
-
       # Wraps the response from Elasticsearch.
       #
       class Response

data/lib/elasticsearch/transport/transport/serializer/multi_json.rb

@@ -6,7 +6,7 @@
 # not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#	http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an

data/lib/elasticsearch/transport/transport/sniffer.rb

@@ -6,7 +6,7 @@
 # not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#	http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
@@ -45,7 +45,8 @@ module Elasticsearch
         #
         def hosts
           Timeout::timeout(timeout, SnifferTimeoutError) do
-            nodes = transport.perform_request('GET', '_nodes/http').body
+            nodes = transport.perform_request('GET', '_nodes/http', {}, nil, nil,
+                                              reload_on_failure: false).body
 
             hosts = nodes['nodes'].map do |id, info|
               if info[PROTOCOL]

data/lib/elasticsearch/transport/version.rb

@@ -6,7 +6,7 @@
 # not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#	http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
@@ -17,6 +17,6 @@
 
 module Elasticsearch
   module Transport
-    VERSION = "7.1.0"
+    VERSION = "7.8.0"
   end
 end

data/lib/elasticsearch/transport.rb

@@ -6,7 +6,7 @@
 # not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#	http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an

data/lib/elasticsearch-transport.rb

@@ -6,7 +6,7 @@
 # not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#	http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an