elasticsearch-transport 7.4.0 → 7.17.10

data/lib/elasticsearch/transport/transport/base.rb

@@ -1,11 +1,23 @@
-# Licensed to Elasticsearch B.V under one or more agreements.
-# Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
-# See the LICENSE file in the project root for more information
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 module Elasticsearch
   module Transport
     module Transport
-
       # @abstract Module with common functionality for transport implementations.
       #
       module Base
@@ -34,13 +46,14 @@ module Elasticsearch
         #
         # @see Client#initialize
         #
-        def initialize(arguments={}, &block)
+        def initialize(arguments = {}, &block)
           @state_mutex = Mutex.new
 
           @hosts       = arguments[:hosts]   || []
           @options     = arguments[:options] || {}
           @options[:http] ||= {}
           @options[:retry_on_status] ||= []
+          @options[:delay_on_retry]  ||= 0
 
           @block       = block
           @compression = !!@options[:compression]
@@ -210,7 +223,7 @@ module Elasticsearch
         # @api private
         #
         def __convert_to_json(o=nil, options={})
-          o = o.is_a?(String) ? o : serializer.dump(o, options)
+          o.is_a?(String) ? o : serializer.dump(o, options)
         end
 
         # Returns a full URL based on information from host
@@ -221,9 +234,9 @@ module Elasticsearch
         def __full_url(host)
           url  = "#{host[:protocol]}://"
           url += "#{CGI.escape(host[:user])}:#{CGI.escape(host[:password])}@" if host[:user]
-          url += "#{host[:host]}"
+          url += host[:host]
           url += ":#{host[:port]}" if host[:port]
-          url += "#{host[:path]}" if host[:path]
+          url += host[:path] if host[:path]
           url
         end
 
@@ -245,11 +258,13 @@ module Elasticsearch
         # @raise  [ServerError]   If request failed on server
         # @raise  [Error]         If no connection is available
         #
-        def perform_request(method, path, params={}, body=nil, headers=nil, opts={}, &block)
-          raise NoMethodError, "Implement this method in your transport class" unless block_given?
+        def perform_request(method, path, params = {}, body = nil, headers = nil, opts = {}, &block)
+          raise NoMethodError, 'Implement this method in your transport class' unless block_given?
+
           start = Time.now
           tries = 0
           reload_on_failure = opts.fetch(:reload_on_failure, @options[:reload_on_failure])
+          delay_on_retry = opts.fetch(:delay_on_retry, @options[:delay_on_retry])
 
           max_retries = if opts.key?(:retry_on_failure)
             opts[:retry_on_failure] === true ? DEFAULT_MAX_RETRIES : opts[:retry_on_failure]
@@ -258,21 +273,19 @@ module Elasticsearch
           end
 
           params = params.clone
-
           ignore = Array(params.delete(:ignore)).compact.map { |s| s.to_i }
 
           begin
+            sleep(delay_on_retry / 1000.0) if tries > 0
             tries     += 1
-            connection = get_connection or raise Error.new("Cannot get new connection from pool.")
+            connection = get_connection or raise Error.new('Cannot get new connection from pool.')
 
             if connection.connection.respond_to?(:params) && connection.connection.params.respond_to?(:to_hash)
               params = connection.connection.params.merge(params.to_hash)
             end
 
-            url        = connection.full_url(path, params)
-
-            response   = block.call(connection, url)
-
+            url      = connection.full_url(path, params)
+            response = block.call(connection, url)
             connection.healthy! if connection.failures > 0
 
             # Raise an exception so we can catch it for `retry_on_status`
@@ -295,7 +308,6 @@ module Elasticsearch
             log_error "[#{e.class}] #{e.message} #{connection.host.inspect}"
 
             connection.dead!
-
             if reload_on_failure and tries < connections.all.size
               log_warn "[#{e.class}] Reloading connections (attempt #{tries} of #{connections.all.size})"
               reload_connections! and retry
@@ -322,14 +334,10 @@ module Elasticsearch
           duration = Time.now - start
 
           if response.status.to_i >= 300
-            __log_response    method, path, params, body, url, response, nil, 'N/A', duration
-            __trace  method, path, params, connection.connection.headers, body, url, response, nil, 'N/A', duration if tracer
-
+            __log_response(method, path, params, body, url, response, nil, 'N/A', duration)
+            __trace(method, path, params, connection_headers(connection), body, url, response, nil, 'N/A', duration) if tracer
             # Log the failure only when `ignore` doesn't match the response status
-            unless ignore.include?(response.status.to_i)
-              log_fatal "[#{response.status}] #{response.body}"
-            end
-
+            log_fatal "[#{response.status}] #{response.body}" unless ignore.include?(response.status.to_i)
             __raise_transport_error response unless ignore.include?(response.status.to_i)
           end
 
@@ -340,8 +348,8 @@ module Elasticsearch
             __log_response   method, path, params, body, url, response, json, took, duration
           end
 
-          __trace  method, path, params, connection.connection.headers, body, url, response, nil, 'N/A', duration if tracer
-
+          __trace(method, path, params, connection_headers(connection), body, url, response, nil, 'N/A', duration) if tracer
+          log_warn(response.headers['warning']) if response.headers&.[]('warning')
           Response.new response.status, json || response.body, response.headers
         ensure
           @last_request_at = Time.now
@@ -360,17 +368,38 @@ module Elasticsearch
 
         USER_AGENT_STR = 'User-Agent'.freeze
         USER_AGENT_REGEX = /user\-?\_?agent/
+        ACCEPT_ENCODING = 'Accept-Encoding'.freeze
+        CONTENT_ENCODING = 'Content-Encoding'.freeze
         CONTENT_TYPE_STR = 'Content-Type'.freeze
         CONTENT_TYPE_REGEX = /content\-?\_?type/
         DEFAULT_CONTENT_TYPE = 'application/json'.freeze
         GZIP = 'gzip'.freeze
-        ACCEPT_ENCODING = 'Accept-Encoding'.freeze
         GZIP_FIRST_TWO_BYTES = '1f8b'.freeze
         HEX_STRING_DIRECTIVE = 'H*'.freeze
         RUBY_ENCODING = '1.9'.respond_to?(:force_encoding)
 
+        def compress_request(body, headers)
+          if body
+            headers ||= {}
+
+            if gzipped?(body)
+              headers[CONTENT_ENCODING] = GZIP
+            elsif use_compression?
+              headers[CONTENT_ENCODING] = GZIP
+              gzip = Zlib::GzipWriter.new(StringIO.new)
+              gzip << body
+              body = gzip.close.string
+            else
+              headers.delete(CONTENT_ENCODING)
+            end
+          elsif headers
+            headers.delete(CONTENT_ENCODING)
+          end
+
+          [body, headers]
+        end
+
         def decompress_response(body)
-          return body unless use_compression?
           return body unless gzipped?(body)
 
           io = StringIO.new(body)
@@ -383,6 +412,8 @@ module Elasticsearch
         end
 
         def gzipped?(body)
+          return unless body && !body.empty?
+
           body[0..1].unpack(HEX_STRING_DIRECTIVE)[0] == GZIP_FIRST_TWO_BYTES
         end
 
@@ -399,7 +430,7 @@ module Elasticsearch
         end
 
         def find_value(hash, regex)
-          key_value = hash.find { |k,v| k.to_s.downcase =~ regex }
+          key_value = hash.find { |k, _| k.to_s.downcase =~ regex }
           if key_value
             hash.delete(key_value[0])
             key_value[1]
@@ -415,7 +446,15 @@ module Elasticsearch
             "elasticsearch-ruby/#{VERSION} (#{meta.join('; ')})"
           end
         end
+
+        def connection_headers(connection)
+          if defined?(Elasticsearch::Transport::Transport::HTTP::Manticore) && self.class == Elasticsearch::Transport::Transport::HTTP::Manticore
+            @request_options[:headers]
+          else
+            connection.connection.headers
+          end
+        end
       end
     end
   end
-end
+end

data/lib/elasticsearch/transport/transport/connections/collection.rb

@@ -1,6 +1,19 @@
-# Licensed to Elasticsearch B.V under one or more agreements.
-# Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
-# See the LICENSE file in the project root for more information
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 module Elasticsearch
   module Transport
@@ -65,16 +78,13 @@ module Elasticsearch
 
           # Returns a connection.
           #
-          # If there are no alive connections, resurrects a connection with least failures.
+          # If there are no alive connections, returns a connection with least failures.
           # Delegates to selector's `#select` method to get the connection.
           #
           # @return [Connection]
           #
           def get_connection(options={})
-            if connections.empty? && dead_connection = dead.sort { |a,b| a.failures <=> b.failures }.first
-              dead_connection.alive!
-            end
-            selector.select(options)
+            selector.select(options) || @connections.min_by(&:failures)
           end
 
           def each(&block)

data/lib/elasticsearch/transport/transport/connections/connection.rb

@@ -1,12 +1,24 @@
-# Licensed to Elasticsearch B.V under one or more agreements.
-# Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
-# See the LICENSE file in the project root for more information
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 module Elasticsearch
   module Transport
     module Transport
       module Connections
-
         # Wraps the connection information and logic.
         #
         # The Connection instance wraps the host information (hostname, port, attributes, etc),
@@ -21,6 +33,7 @@ module Elasticsearch
           DEFAULT_RESURRECT_TIMEOUT = 60
 
           attr_reader :host, :connection, :options, :failures, :dead_since
+          attr_accessor :verified
 
           # @option arguments [Hash]   :host       Host information (example: `{host: 'localhost', port: 9200}`)
           # @option arguments [Object] :connection The transport-specific physical connection or "session"
@@ -30,6 +43,7 @@ module Elasticsearch
             @host       = arguments[:host].is_a?(Hash) ? Redacted.new(arguments[:host]) : arguments[:host]
             @connection = arguments[:connection]
             @options    = arguments[:options] || {}
+            @verified   = false
             @state_mutex = Mutex.new
 
             @options[:resurrect_timeout] ||= DEFAULT_RESURRECT_TIMEOUT
@@ -41,12 +55,14 @@ module Elasticsearch
           #
           # @return [String]
           #
-          def full_url(path, params={})
+          def full_url(path, params = {})
             url  = "#{host[:protocol]}://"
             url += "#{CGI.escape(host[:user])}:#{CGI.escape(host[:password])}@" if host[:user]
             url += "#{host[:host]}:#{host[:port]}"
             url += "#{host[:path]}" if host[:path]
-            url += "/#{full_path(path, params)}"
+            full_path = full_path(path, params)
+            url += '/' unless full_path.match?(/^\//)
+            url += full_path
           end
 
           # Returns the complete endpoint path with serialized parameters.
@@ -122,14 +138,15 @@ module Elasticsearch
             }
           end
 
-          # Equality operator based on connection protocol, host and port
+          # Equality operator based on connection protocol, host, port and attributes
           #
           # @return [Boolean]
           #
           def ==(other)
             self.host[:protocol] == other.host[:protocol] && \
             self.host[:host] == other.host[:host] && \
-            self.host[:port].to_i == other.host[:port].to_i
+            self.host[:port].to_i == other.host[:port].to_i && \
+            self.host[:attributes] == other.host[:attributes]
           end
 
           # @return [String]
@@ -138,7 +155,6 @@ module Elasticsearch
             "<#{self.class.name} host: #{host} (#{dead? ? 'dead since ' + dead_since.to_s : 'alive'})>"
           end
         end
-
       end
     end
   end

data/lib/elasticsearch/transport/transport/connections/selector.rb

@@ -1,6 +1,19 @@
-# Licensed to Elasticsearch B.V under one or more agreements.
-# Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
-# See the LICENSE file in the project root for more information
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 module Elasticsearch
   module Transport

data/lib/elasticsearch/transport/transport/errors.rb

@@ -1,6 +1,19 @@
-# Licensed to Elasticsearch B.V under one or more agreements.
-# Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
-# See the LICENSE file in the project root for more information
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 module Elasticsearch
   module Transport
@@ -51,6 +64,7 @@ module Elasticsearch
         418 => 'ImATeapot',
         421 => 'TooManyConnectionsFromThisIP',
         426 => 'UpgradeRequired',
+        429 => 'TooManyRequests',
         450 => 'BlockedByWindowsParentalControls',
         494 => 'RequestHeaderTooLarge',
         497 => 'HTTPToHTTPS',

data/lib/elasticsearch/transport/transport/http/curb.rb

@@ -1,56 +1,82 @@
-# Licensed to Elasticsearch B.V under one or more agreements.
-# Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
-# See the LICENSE file in the project root for more information
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 module Elasticsearch
   module Transport
     module Transport
       module HTTP
-
         # Alternative HTTP transport implementation, using the [_Curb_](https://rubygems.org/gems/curb) client.
         #
         # @see Transport::Base
         #
         class Curb
           include Base
-
           # Performs the request by invoking {Transport::Base#perform_request} with a block.
           #
           # @return [Response]
           # @see    Transport::Base#perform_request
           #
           def perform_request(method, path, params={}, body=nil, headers=nil, opts={})
-            super do |connection, url|
+            super do |connection, _url|
               connection.connection.url = connection.full_url(path, params)
+              body = body ? __convert_to_json(body) : nil
+              body, headers = compress_request(body, headers)
 
               case method
-                when 'HEAD'
-                  connection.connection.set :nobody, true
-                when 'GET', 'POST', 'PUT', 'DELETE'
-                  connection.connection.set :nobody, false
-
-                  connection.connection.put_data = __convert_to_json(body) if body
-
-                  if headers
-                    if connection.connection.headers
-                      connection.connection.headers.merge!(headers)
-                    else
-                      connection.connection.headers = headers
-                    end
+              when 'HEAD'
+                connection.connection.set :nobody, true
+              when 'GET', 'POST', 'PUT', 'DELETE'
+                connection.connection.set :nobody, false
+
+                connection.connection.put_data = body if body
+
+                if headers
+                  if connection.connection.headers
+                    connection.connection.headers.merge!(headers)
+                  else
+                    connection.connection.headers = headers
                   end
-
-                else raise ArgumentError, "Unsupported HTTP method: #{method}"
+                end
+              else
+                raise ArgumentError, "Unsupported HTTP method: #{method}"
               end
 
               connection.connection.http(method.to_sym)
 
-              response_headers = {}
-              response_headers['content-type'] = 'application/json' if connection.connection.header_str =~ /\/json/
+              Response.new(
+                connection.connection.response_code,
+                decompress_response(connection.connection.body_str),
+                headers(connection)
+              )
+            end
+          end
+
+          def headers(connection)
+            headers_string = connection.connection.header_str
+            return nil if headers_string.nil?
 
-              Response.new connection.connection.response_code,
-                           decompress_response(connection.connection.body_str),
-                           response_headers
+            response_headers = headers_string&.split(/\\r\\n|\r\n/).reject(&:empty?)
+            response_headers.shift # Removes HTTP status string
+            processed_header = response_headers.flat_map { |s| s.scan(/^(\S+): (.+)/) }
+            headers_hash = Hash[processed_header].transform_keys(&:downcase)
+            if headers_hash['content-type']&.match?(/application\/json/)
+              headers_hash['content-type'] = 'application/json'
             end
+            headers_hash
           end
 
           # Builds and returns a connection
@@ -59,9 +85,8 @@ module Elasticsearch
           #
           def __build_connection(host, options={}, block=nil)
             client = ::Curl::Easy.new
-
             apply_headers(client, options)
-            client.url     = __full_url(host)
+            client.url = __full_url(host)
 
             if host[:user]
               client.http_auth_types = host[:auth_type] || :basic
@@ -93,13 +118,13 @@ module Elasticsearch
 
           def user_agent_header(client)
             @user_agent ||= begin
-              meta = ["RUBY_VERSION: #{RUBY_VERSION}"]
-              if RbConfig::CONFIG && RbConfig::CONFIG['host_os']
-                meta << "#{RbConfig::CONFIG['host_os'].split('_').first[/[a-z]+/i].downcase} #{RbConfig::CONFIG['target_cpu']}"
-              end
-              meta << "Curb #{Curl::CURB_VERSION}"
-              "elasticsearch-ruby/#{VERSION} (#{meta.join('; ')})"
-            end
+                              meta = ["RUBY_VERSION: #{RUBY_VERSION}"]
+                              if RbConfig::CONFIG && RbConfig::CONFIG['host_os']
+                                meta << "#{RbConfig::CONFIG['host_os'].split('_').first[/[a-z]+/i].downcase} #{RbConfig::CONFIG['target_cpu']}"
+                              end
+                              meta << "Curb #{Curl::CURB_VERSION}"
+                              "elasticsearch-ruby/#{VERSION} (#{meta.join('; ')})"
+                            end
           end
         end
       end

data/lib/elasticsearch/transport/transport/http/faraday.rb

@@ -1,12 +1,24 @@
-# Licensed to Elasticsearch B.V under one or more agreements.
-# Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
-# See the LICENSE file in the project root for more information
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 module Elasticsearch
   module Transport
     module Transport
       module HTTP
-
         # The default transport implementation, using the [_Faraday_](https://rubygems.org/gems/faraday)
         # library for abstracting the HTTP client.
         #
@@ -20,13 +32,24 @@ module Elasticsearch
           # @return [Response]
           # @see    Transport::Base#perform_request
           #
-          def perform_request(method, path, params={}, body=nil, headers=nil, opts={})
+          def perform_request(method, path, params = {}, body = nil, headers = nil, opts = {})
             super do |connection, url|
-              headers = headers || connection.connection.headers
+              headers = if connection.connection.headers
+                          if !headers.nil?
+                            connection.connection.headers.merge(headers)
+                          else
+                            connection.connection.headers
+                          end
+                        else
+                          headers
+                        end
+
+              body = body ? __convert_to_json(body) : nil
+              body, headers = compress_request(body, headers)
 
               response = connection.connection.run_request(method.downcase.to_sym,
                                                            url,
-                                                           ( body ? __convert_to_json(body) : nil ),
+                                                           body,
                                                            headers)
 
               Response.new response.status, decompress_response(response.body), response.headers
@@ -40,7 +63,7 @@ module Elasticsearch
           def __build_connection(host, options={}, block=nil)
             client = ::Faraday.new(__full_url(host), options, &block)
             apply_headers(client, options)
-            Connections::Connection.new :host => host, :connection => client
+            Connections::Connection.new(host: host, connection: client)
           end
 
           # Returns an array of implementation specific connection errors.
@@ -48,7 +71,7 @@ module Elasticsearch
           # @return [Array]
           #
           def host_unreachable_exceptions
-            [::Faraday::Error::ConnectionFailed, ::Faraday::Error::TimeoutError]
+            [::Faraday::ConnectionFailed, ::Faraday::TimeoutError]
           end
 
           private