elasticsearch-transport 7.1.0 → 7.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 14188a454b64314c13f7c9279741fc8df5f9f03ff9813f45aeca96abc8da7e39
-  data.tar.gz: dfcd7ac5fb8e0b2a0450b2939272f88a329581648c47b130d62e1fcead705b44
+  metadata.gz: 47a8f6e85b60bdec378a1c8011aa76c24ab1a08f618a2f2fb38850f3b6114190
+  data.tar.gz: 25e88a3e742a0e4341ba685296425d13ddb8ed7059ef97a39174e394bec40535
 SHA512:
-  metadata.gz: 36e9583f541ec75994cb27479c16f4624d7ea6a9e6f33360d7cac69541fe10a5c2ee2d7f20b2c1231b5fd07ab85cd32baefb236d6ab77ff11635bb23b1d9d801
-  data.tar.gz: 95faf1016519c04dcafcd6580356dcfb00c8eeb9ef3301bf2ff3c66a90cfcc8ebf676b638ec18f92f708acb5ef01ee2f7ee137f23ed6532774b371e47b02e11f
+  metadata.gz: 90a600694fa1172722d988da7cf09daca12160054b13c1b6a8093303529f62fcd76eb8c4367c0f936fb9d96fdc0533e849ed7c426b04f40069cf009895c06da4
+  data.tar.gz: 6f18333cf05353ed2292c7edf21cc320aa8482f72e07bfab104e7ca8b5d0d9d697f26b2562da8455852dd67afe3e1a197edf70133527287574441e85da08b566

data/README.md

@@ -106,6 +106,17 @@ Another way to configure the URL(s) is to export the `ELASTICSEARCH_URL` variabl
 The client will automatically round-robin across the hosts
 (unless you select or implement a different [connection selector](#connection-selector)).
 
+### Connect using an Elastic Cloud ID
+
+If you are using [Elastic Cloud](https://www.elastic.co/cloud), you can provide your cloud id to the client.
+You must supply your username and password separately, and optionally a port. If no port is supplied,
+port 9243 will be used.
+
+Note: Do not enable sniffing when using Elastic Cloud. The nodes are behind a load balancer so
+Elastic Cloud will take care of everything for you.
+
+    Elasticsearch::Client.new(cloud_id: 'name:bG9jYWxob3N0JGFiY2QkZWZnaA==', user: 'elastic', password: 'changeme')
+
 ### Authentication
 
 You can pass the authentication credentials, scheme and port in the host configuration hash:

data/elasticsearch-transport.gemspec

@@ -65,6 +65,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency "patron" unless defined? JRUBY_VERSION
   s.add_development_dependency "typhoeus", '~> 0.6'
   s.add_development_dependency "net-http-persistent"
+  s.add_development_dependency "httpclient"
   s.add_development_dependency "manticore", '~> 0.6' if defined? JRUBY_VERSION
   s.add_development_dependency "hashie"
 

data/lib/elasticsearch/transport/client.rb

@@ -15,6 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+require 'base64'
+
 module Elasticsearch
   module Transport
 
@@ -46,6 +48,11 @@ module Elasticsearch
       # @since 7.0.0
       DEFAULT_HOST = 'localhost:9200'.freeze
 
+      # The default port to use if connecting using a Cloud ID.
+      #
+      # @since 7.2.0
+      DEFAULT_CLOUD_PORT = 9243
+
       # Returns the transport object.
       #
       # @see Elasticsearch::Transport::Transport::Base
@@ -101,6 +108,9 @@ module Elasticsearch
       #
       # @option arguments [String] :send_get_body_as Specify the HTTP method to use for GET requests with a body.
       #                                              (Default: GET)
+      # @option arguments [true, false] :compression Whether to compress requests. Gzip compression will be used.
+      #   The default is false. Responses will automatically be inflated if they are compressed.
+      #   If a custom transport object is used, it must handle the request compression and response inflation.
       #
       # @yield [faraday] Access and configure the `Faraday::Connection` instance directly with a block
       #
@@ -117,7 +127,8 @@ module Elasticsearch
         @arguments[:http]               ||= {}
         @options[:http]               ||= {}
 
-        @seeds = __extract_hosts(@arguments[:hosts] ||
+        @seeds = extract_cloud_creds(@arguments)
+        @seeds ||= __extract_hosts(@arguments[:hosts] ||
                                      @arguments[:host] ||
                                      @arguments[:url] ||
                                      @arguments[:urls] ||
@@ -130,12 +141,6 @@ module Elasticsearch
           @arguments[:transport_options][:request] = { :timeout => @arguments[:request_timeout] }
         end
 
-        @arguments[:transport_options][:headers] ||= {}
-
-        unless @arguments[:transport_options][:headers].keys.any? {|k| k.to_s.downcase =~ /content\-?\_?type/}
-          @arguments[:transport_options][:headers]['Content-Type'] = 'application/json'
-        end
-
         if @arguments[:transport]
           @transport = @arguments[:transport]
         else
@@ -162,6 +167,16 @@ module Elasticsearch
 
       private
 
+      def extract_cloud_creds(arguments)
+        return unless arguments[:cloud_id]
+        cloud_url, elasticsearch_instance = Base64.decode64(arguments[:cloud_id].gsub('name:', '')).split('$')
+        [ { scheme: 'https',
+            user: arguments[:user],
+            password: arguments[:password],
+            host: "#{elasticsearch_instance}.#{cloud_url}",
+            port: arguments[:port] || DEFAULT_CLOUD_PORT } ]
+      end
+
       # Normalizes and returns hosts configuration.
       #
       # Arrayifies the `hosts_config` argument and extracts `host` and `port` info from strings.

data/lib/elasticsearch/transport/transport/base.rb

@@ -56,6 +56,7 @@ module Elasticsearch
           @options[:retry_on_status] ||= []
 
           @block       = block
+          @compression = !!@options[:compression]
           @connections = __build_connections
 
           @serializer  = options[:serializer] || ( options[:serializer_class] ? options[:serializer_class].new(self) : DEFAULT_SERIALIZER_CLASS.new(self) )
@@ -202,7 +203,7 @@ module Elasticsearch
               ( params.empty? ? '' : "&#{::Faraday::Utils::ParamsHash[params].to_query}" )
           trace_body = body ? " -d '#{__convert_to_json(body, :pretty => true)}'" : ''
           trace_command = "curl -X #{method.to_s.upcase}"
-          trace_command += " -H '#{headers.inject('') { |memo,item| memo << item[0] + ': ' + item[1] }}'" if headers && !headers.empty?
+          trace_command += " -H '#{headers.collect { |k,v| "#{k}: #{v}" }.join(", ")}'" if headers && !headers.empty?
           trace_command += " '#{trace_url}'#{trace_body}\n"
           tracer.info trace_command
           tracer.debug "# #{Time.now.iso8601} [#{response.status}] (#{format('%.3f', duration)}s)\n#"
@@ -234,7 +235,8 @@ module Elasticsearch
         def __full_url(host)
           url  = "#{host[:protocol]}://"
           url += "#{CGI.escape(host[:user])}:#{CGI.escape(host[:password])}@" if host[:user]
-          url += "#{host[:host]}:#{host[:port]}"
+          url += "#{host[:host]}"
+          url += ":#{host[:port]}" if host[:port]
           url += "#{host[:path]}" if host[:path]
           url
         end
@@ -328,7 +330,7 @@ module Elasticsearch
 
           if response.status.to_i >= 300
             __log_response    method, path, params, body, url, response, nil, 'N/A', duration
-            __trace  method, path, params, headers, body, url, response, nil, 'N/A', duration if tracer
+            __trace  method, path, params, connection.connection.headers, body, url, response, nil, 'N/A', duration if tracer
 
             # Log the failure only when `ignore` doesn't match the response status
             unless ignore.include?(response.status.to_i)
@@ -345,7 +347,7 @@ module Elasticsearch
             __log_response   method, path, params, body, url, response, json, took, duration
           end
 
-          __trace method, path, params, headers, body, url, response, json, took, duration if tracer
+          __trace  method, path, params, connection.connection.headers, body, url, response, nil, 'N/A', duration if tracer
 
           Response.new response.status, json || response.body, response.headers
         ensure
@@ -360,6 +362,66 @@ module Elasticsearch
         def host_unreachable_exceptions
           [Errno::ECONNREFUSED]
         end
+
+        private
+
+        USER_AGENT_STR = 'User-Agent'.freeze
+        USER_AGENT_REGEX = /user\-?\_?agent/
+        CONTENT_TYPE_STR = 'Content-Type'.freeze
+        CONTENT_TYPE_REGEX = /content\-?\_?type/
+        DEFAULT_CONTENT_TYPE = 'application/json'.freeze
+        GZIP = 'gzip'.freeze
+        ACCEPT_ENCODING = 'Accept-Encoding'.freeze
+        GZIP_FIRST_TWO_BYTES = '1f8b'.freeze
+        HEX_STRING_DIRECTIVE = 'H*'.freeze
+        RUBY_ENCODING = '1.9'.respond_to?(:force_encoding)
+
+        def decompress_response(body)
+          return body unless use_compression?
+          return body unless gzipped?(body)
+
+          io = StringIO.new(body)
+          gzip_reader = if RUBY_ENCODING
+                          Zlib::GzipReader.new(io, :encoding => 'ASCII-8BIT')
+                        else
+                          Zlib::GzipReader.new(io)
+                        end
+          gzip_reader.read
+        end
+
+        def gzipped?(body)
+          body[0..1].unpack(HEX_STRING_DIRECTIVE)[0] == GZIP_FIRST_TWO_BYTES
+        end
+
+        def use_compression?
+          @compression
+        end
+
+        def apply_headers(client, options)
+          headers = options[:headers] || {}
+          headers[CONTENT_TYPE_STR] = find_value(headers, CONTENT_TYPE_REGEX) || DEFAULT_CONTENT_TYPE
+          headers[USER_AGENT_STR] = find_value(headers, USER_AGENT_REGEX) || user_agent_header(client)
+          client.headers[ACCEPT_ENCODING] = GZIP if use_compression?
+          client.headers.merge!(headers)
+        end
+
+        def find_value(hash, regex)
+          key_value = hash.find { |k,v| k.to_s.downcase =~ regex }
+          if key_value
+            hash.delete(key_value[0])
+            key_value[1]
+          end
+        end
+
+        def user_agent_header(client)
+          @user_agent ||= begin
+            meta = ["RUBY_VERSION: #{RUBY_VERSION}"]
+            if RbConfig::CONFIG && RbConfig::CONFIG['host_os']
+              meta << "#{RbConfig::CONFIG['host_os'].split('_').first[/[a-z]+/i].downcase} #{RbConfig::CONFIG['target_cpu']}"
+            end
+            "elasticsearch-ruby/#{VERSION} (#{meta.join('; ')})"
+          end
+        end
       end
     end
   end

data/lib/elasticsearch/transport/transport/connections/selector.rb

@@ -61,18 +61,30 @@ module Elasticsearch
           class RoundRobin
             include Base
 
+            # @option arguments [Connections::Collection] :connections Collection with connections.
+            #
+            def initialize(arguments = {})
+              super
+              @mutex = Mutex.new
+              @current = nil
+            end
+
             # Returns the next connection from the collection, rotating them in round-robin fashion.
             #
             # @return [Connections::Connection]
             #
             def select(options={})
-              # On Ruby 1.9, Array#rotate could be used instead
-              @current = !defined?(@current) || @current.nil? ? 0 : @current+1
-              @current = 0 if @current >= connections.size
-              connections[@current]
+              @mutex.synchronize do
+                conns = connections
+                if @current && (@current < conns.size-1)
+                  @current += 1
+                else
+                  @current = 0
+                end
+                conns[@current]
+              end
             end
           end
-
         end
       end
     end

data/lib/elasticsearch/transport/transport/http/curb.rb

@@ -43,7 +43,15 @@ module Elasticsearch
                   connection.connection.set :nobody, false
 
                   connection.connection.put_data = __convert_to_json(body) if body
-                  connection.connection.headers = headers if headers
+
+                  if headers
+                    if connection.connection.headers
+                      connection.connection.headers.merge!(headers)
+                    else
+                      connection.connection.headers = headers
+                    end
+                  end
+
                 else raise ArgumentError, "Unsupported HTTP method: #{method}"
               end
 
@@ -53,7 +61,7 @@ module Elasticsearch
               response_headers['content-type'] = 'application/json' if connection.connection.header_str =~ /\/json/
 
               Response.new connection.connection.response_code,
-                           connection.connection.body_str,
+                           decompress_response(connection.connection.body_str),
                            response_headers
             end
           end
@@ -65,10 +73,7 @@ module Elasticsearch
           def __build_connection(host, options={}, block=nil)
             client = ::Curl::Easy.new
 
-            headers = options[:headers] || {}
-            headers.update('User-Agent' => "Curb #{Curl::CURB_VERSION}")
-
-            client.headers = headers
+            apply_headers(client, options)
             client.url     = __full_url(host)
 
             if host[:user]
@@ -96,8 +101,20 @@ module Elasticsearch
               ::Curl::Err::TimeoutError
             ]
           end
-        end
 
+          private
+
+          def user_agent_header(client)
+            @user_agent ||= begin
+              meta = ["RUBY_VERSION: #{RUBY_VERSION}"]
+              if RbConfig::CONFIG && RbConfig::CONFIG['host_os']
+                meta << "#{RbConfig::CONFIG['host_os'].split('_').first[/[a-z]+/i].downcase} #{RbConfig::CONFIG['target_cpu']}"
+              end
+              meta << "Curb #{Curl::CURB_VERSION}"
+              "elasticsearch-ruby/#{VERSION} (#{meta.join('; ')})"
+            end
+          end
+        end
       end
     end
   end

data/lib/elasticsearch/transport/transport/http/faraday.rb

@@ -42,7 +42,7 @@ module Elasticsearch
                                                            ( body ? __convert_to_json(body) : nil ),
                                                            headers)
 
-              Response.new response.status, response.body, response.headers
+              Response.new response.status, decompress_response(response.body), response.headers
             end
           end
 
@@ -52,6 +52,7 @@ module Elasticsearch
           #
           def __build_connection(host, options={}, block=nil)
             client = ::Faraday.new(__full_url(host), options, &block)
+            apply_headers(client, options)
             Connections::Connection.new :host => host, :connection => client
           end
 
@@ -62,6 +63,19 @@ module Elasticsearch
           def host_unreachable_exceptions
             [::Faraday::Error::ConnectionFailed, ::Faraday::Error::TimeoutError]
           end
+
+          private
+
+          def user_agent_header(client)
+            @user_agent ||= begin
+              meta = ["RUBY_VERSION: #{RUBY_VERSION}"]
+              if RbConfig::CONFIG && RbConfig::CONFIG['host_os']
+                meta << "#{RbConfig::CONFIG['host_os'].split('_').first[/[a-z]+/i].downcase} #{RbConfig::CONFIG['target_cpu']}"
+              end
+              meta << "#{client.headers[USER_AGENT_STR]}"
+              "elasticsearch-ruby/#{VERSION} (#{meta.join('; ')})"
+            end
+          end
         end
       end
     end

data/lib/elasticsearch/transport/transport/http/manticore.rb

@@ -110,14 +110,8 @@ module Elasticsearch
           #
           def __build_connections
             @request_options = {}
-
-            if options[:transport_options] && options[:transport_options][:headers]
-              @request_options[:headers] = options[:transport_options][:headers]
-            end
-
-            if options.key?(:headers)
-              @request_options[:headers] = options[:headers]
-            end
+            apply_headers(@request_options, options[:transport_options])
+            apply_headers(@request_options, options)
 
             Connections::Collection.new \
               :connections => hosts.map { |host|
@@ -157,6 +151,27 @@ module Elasticsearch
               ::Manticore::ResolutionFailure
             ]
           end
+
+          private
+
+          def apply_headers(request_options, options)
+            headers = (options && options[:headers]) || {}
+            headers[CONTENT_TYPE_STR] = find_value(headers, CONTENT_TYPE_REGEX) || DEFAULT_CONTENT_TYPE
+            headers[USER_AGENT_STR] = find_value(headers, USER_AGENT_REGEX) || user_agent_header
+            headers[ACCEPT_ENCODING] = GZIP if use_compression?
+            request_options.merge!(headers: headers)
+          end
+
+          def user_agent_header
+            @user_agent ||= begin
+              meta = ["RUBY_VERSION: #{JRUBY_VERSION}"]
+              if RbConfig::CONFIG && RbConfig::CONFIG['host_os']
+                meta << "#{RbConfig::CONFIG['host_os'].split('_').first[/[a-z]+/i].downcase} #{RbConfig::CONFIG['target_cpu']}"
+              end
+              meta << "Manticore #{::Manticore::VERSION}"
+              "elasticsearch-ruby/#{VERSION} (#{meta.join('; ')})"
+            end
+          end
         end
       end
     end

data/lib/elasticsearch/transport/version.rb

@@ -17,6 +17,6 @@
 
 module Elasticsearch
   module Transport
-    VERSION = "7.1.0"
+    VERSION = "7.2.0"
   end
 end

data/spec/elasticsearch/connections/collection_spec.rb

@@ -0,0 +1,254 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require 'spec_helper'
+
+describe Elasticsearch::Transport::Transport::Connections::Collection do
+
+  describe '#initialize' do
+
+    let(:collection) do
+      described_class.new
+    end
+
+    it 'has an empty list of connections as a default' do
+      expect(collection.connections).to be_empty
+    end
+
+    it 'has a default selector class' do
+      expect(collection.selector).not_to be_nil
+    end
+
+    context 'when a selector class is specified' do
+
+      let(:collection) do
+        described_class.new(selector_class: Elasticsearch::Transport::Transport::Connections::Selector::Random)
+      end
+
+      it 'sets the selector' do
+        expect(collection.selector).to be_a(Elasticsearch::Transport::Transport::Connections::Selector::Random)
+      end
+    end
+  end
+
+  describe '#get_connection' do
+
+    let(:collection) do
+      described_class.new(selector_class: Elasticsearch::Transport::Transport::Connections::Selector::Random)
+    end
+
+    before do
+      expect(collection.selector).to receive(:select).and_return('OK')
+    end
+
+    it 'uses the selector to select a connection' do
+      expect(collection.get_connection).to eq('OK')
+    end
+  end
+
+  describe '#hosts' do
+
+    let(:collection) do
+      described_class.new(connections: [ double('connection', host: 'A'),
+                                         double('connection', host: 'B') ])
+    end
+
+    it 'returns a list of hosts' do
+      expect(collection.hosts).to eq([ 'A', 'B'])
+    end
+  end
+
+  describe 'enumerable' do
+
+    let(:collection) do
+      described_class.new(connections: [ double('connection', host: 'A', dead?: false),
+                                         double('connection', host: 'B', dead?: false) ])
+    end
+
+    describe '#map' do
+
+      it 'responds to the method' do
+        expect(collection.map { |c| c.host.downcase }).to eq(['a', 'b'])
+      end
+    end
+
+    describe '#[]' do
+
+      it 'responds to the method' do
+        expect(collection[0].host).to eq('A')
+        expect(collection[1].host).to eq('B')
+      end
+    end
+
+    describe '#size' do
+
+      it 'responds to the method' do
+        expect(collection.size).to eq(2)
+      end
+    end
+
+    context 'when a connection is marked as dead' do
+
+      let(:collection) do
+        described_class.new(connections: [ double('connection', host: 'A', dead?: true),
+                                           double('connection', host: 'B', dead?: false) ])
+      end
+
+      it 'does not enumerate the dead connections' do
+        expect(collection.size).to eq(1)
+        expect(collection.collect { |c| c.host }).to eq(['B'])
+      end
+
+      context '#alive' do
+
+        it 'enumerates the alive connections' do
+          expect(collection.alive.collect { |c| c.host }).to eq(['B'])
+        end
+      end
+
+      context '#dead' do
+
+        it 'enumerates the alive connections' do
+          expect(collection.dead.collect { |c| c.host }).to eq(['A'])
+        end
+      end
+    end
+  end
+
+  describe '#add' do
+
+    let(:collection) do
+      described_class.new(connections: [ double('connection', host: 'A', dead?: false),
+                                         double('connection', host: 'B', dead?: false) ])
+    end
+
+    context 'when an array is provided' do
+
+      before do
+        collection.add([double('connection', host: 'C', dead?: false),
+                        double('connection', host: 'D', dead?: false)])
+      end
+
+      it 'adds the connections' do
+        expect(collection.size).to eq(4)
+      end
+    end
+
+    context 'when an element is provided' do
+
+      before do
+        collection.add(double('connection', host: 'C', dead?: false))
+      end
+
+      it 'adds the connection' do
+        expect(collection.size).to eq(3)
+      end
+    end
+  end
+
+  describe '#remove' do
+
+    let(:connections) do
+      [ double('connection', host: 'A', dead?: false),
+        double('connection', host: 'B', dead?: false) ]
+    end
+
+    let(:collection) do
+      described_class.new(connections: connections)
+    end
+
+    context 'when an array is provided' do
+
+      before do
+        collection.remove(connections)
+      end
+
+      it 'removes the connections' do
+        expect(collection.size).to eq(0)
+      end
+    end
+
+    context 'when an element is provided' do
+
+      let(:connections) do
+        [ double('connection', host: 'A', dead?: false),
+          double('connection', host: 'B', dead?: false) ]
+      end
+
+      before do
+        collection.remove(connections.first)
+      end
+
+      it 'removes the connection' do
+        expect(collection.size).to eq(1)
+      end
+    end
+  end
+
+  describe '#get_connection' do
+
+    context 'when all connections are dead' do
+
+      let(:connection_a) do
+        Elasticsearch::Transport::Transport::Connections::Connection.new(host: { host: 'A' })
+      end
+
+      let(:connection_b) do
+        Elasticsearch::Transport::Transport::Connections::Connection.new(host: { host: 'B' })
+      end
+
+      let(:collection) do
+        described_class.new(connections: [connection_a, connection_b])
+      end
+
+      before do
+        connection_a.dead!.dead!
+        connection_b.dead!
+      end
+
+      it 'returns the connection with the least failures' do
+        expect(collection.get_connection.host[:host]).to eq('B')
+      end
+    end
+
+    context 'when multiple threads are used' do
+
+      let(:connections) do
+        20.times.collect do |i|
+          Elasticsearch::Transport::Transport::Connections::Connection.new(host: { host: i })
+        end
+      end
+
+      let(:collection) do
+        described_class.new(connections: connections)
+      end
+
+      it 'allows threads to select connections in parallel' do
+       expect(10.times.collect do
+          threads = []
+          20.times do
+            threads << Thread.new do
+              collection.get_connection
+            end
+          end
+          threads.map { |t| t.join }
+          collection.get_connection.host[:host]
+        end).to eq((0..9).to_a)
+      end
+    end
+  end
+end