elastic-transport 8.0.0.pre1

data/Rakefile

@@ -0,0 +1,87 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require 'bundler/gem_tasks'
+require 'mkmf'
+
+desc "Run unit tests"
+task default: 'test:unit'
+task test: 'test:unit'
+
+# ----- Test tasks ------------------------------------------------------------
+require 'rake/testtask'
+require 'rspec/core/rake_task'
+
+namespace :test do
+  RSpec::Core::RakeTask.new(:spec)
+
+  Rake::TestTask.new(:unit) do |test|
+    test.libs << 'lib' << 'test'
+    test.test_files = FileList['test/unit/**/*_test.rb']
+    test.verbose = false
+    test.warning = false
+  end
+
+  Rake::TestTask.new(:integration) do |test|
+    test.libs << 'lib' << 'test'
+    test.test_files = FileList['test/integration/**/*_test.rb']
+    test.verbose = false
+    test.warning = false
+  end
+
+  desc 'Run all tests'
+  task :all do
+    Rake::Task['test:unit'].invoke
+    Rake::Task['test:integration'].invoke
+  end
+
+  Rake::TestTask.new(:profile) do |test|
+    test.libs << 'lib' << 'test'
+    test.test_files = FileList['test/profile/**/*_test.rb']
+  end
+end
+
+
+namespace :docker do
+  desc <<~DOC
+    Start Elasticsearch in a Docker container. Credentials are 'elastic:changeme'.
+
+    Default:
+      rake docker:start[version]
+    E.g.:
+      rake docker:start[8.0.0-SNAPSHOT]
+  DOC
+  task :start, [:version] do |_, params|
+    abort 'Docker not installed' unless find_executable 'docker'
+    abort 'You need to set a version, e.g. rake docker:start[7.x-SNAPSHOT]' unless params[:version]
+
+    system("docker run -p 9200:9200 -p 9300:9300 -e 'discovery.type=single-node' -e ELASTIC_PASSWORD=changeme docker.elastic.co/elasticsearch/elasticsearch:#{params[:version]}")
+  end
+end
+
+# ----- Documentation tasks ---------------------------------------------------
+require 'yard'
+YARD::Rake::YardocTask.new(:doc) do |t|
+  t.options = %w| --embed-mixins --markup=markdown |
+end
+
+# ----- Code analysis tasks ---------------------------------------------------
+require 'cane/rake_task'
+Cane::RakeTask.new(:quality) do |cane|
+  cane.abc_max = 15
+  cane.no_style = true
+end

data/elastic-transport.gemspec

@@ -0,0 +1,74 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'elastic/transport/version'
+
+Gem::Specification.new do |s|
+  s.name          = "elastic-transport"
+  s.version       = Elastic::Transport::VERSION
+  s.authors       = ['Karel Minarik', 'Emily Stolfo', 'Fernando Briano']
+  s.email         = ['support@elastic.co']
+  s.summary       = 'Low level Ruby client for Elastic services.'
+  s.homepage      = 'https://github.com/elastic/elastic-transport-ruby'
+  s.license       = 'Apache-2.0'
+  s.metadata = {
+    'homepage_uri' => 'https://github.com/elastic/elastic-transport-ruby',
+    'changelog_uri' => 'https://github.com/elastic/elastic-transport-ruby/blob/master/CHANGELOG.md',
+    'source_code_uri' => 'https://github.com/elastic/elastic-transport-ruby',
+    'bug_tracker_uri' => 'https://github.com/elastic/elastic-transport-ruby/issues'
+  }
+  s.files         = `git ls-files`.split($/)
+  s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+  s.require_paths = ['lib']
+
+  s.extra_rdoc_files  = ['README.md', 'LICENSE']
+  s.rdoc_options      = [ '--charset=UTF-8' ]
+
+  s.required_ruby_version = '>= 2.5'
+
+  s.add_dependency 'multi_json'
+  s.add_dependency 'faraday', '~> 1'
+
+  s.add_development_dependency 'bundler'
+  s.add_development_dependency 'cane'
+  s.add_development_dependency 'curb'   unless defined? JRUBY_VERSION
+  s.add_development_dependency 'hashie'
+  s.add_development_dependency 'httpclient'
+  s.add_development_dependency 'manticore' if defined? JRUBY_VERSION
+  s.add_development_dependency 'minitest'
+  s.add_development_dependency 'minitest-reporters'
+  s.add_development_dependency 'mocha'
+  s.add_development_dependency 'net-http-persistent'
+  s.add_development_dependency 'patron' unless defined? JRUBY_VERSION
+  s.add_development_dependency 'pry'
+  s.add_development_dependency 'rake', '~> 13'
+  s.add_development_dependency 'require-prof' unless defined?(JRUBY_VERSION) || defined?(Rubinius)
+  s.add_development_dependency 'ruby-prof'    unless defined?(JRUBY_VERSION) || defined?(Rubinius)
+  s.add_development_dependency 'shoulda-context'
+  s.add_development_dependency 'simplecov'
+  s.add_development_dependency 'test-unit', '~> 2'
+  s.add_development_dependency 'typhoeus', '~> 1.4'
+  s.add_development_dependency 'yard'
+
+  s.description = <<-DESC.gsub(/^    /, '')
+    Low level Ruby client for Elastic. See the `elasticsearch` or `elastic-enterprise-search` gems for full integration.
+  DESC
+end

data/lib/elastic/transport/client.rb

@@ -0,0 +1,276 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require 'base64'
+require 'elastic/transport/meta_header'
+
+module Elastic
+  module Transport
+    # Handles communication with an Elastic cluster.
+    #
+    # See {file:README.md README} for usage and code examples.
+    #
+    class Client
+      DEFAULT_TRANSPORT_CLASS = Transport::HTTP::Faraday
+      include MetaHeader
+
+      DEFAULT_LOGGER = lambda do
+        require 'logger'
+        logger = Logger.new(STDERR)
+        logger.progname = 'elastic'
+        logger.formatter = proc { |severity, datetime, progname, msg| "#{datetime}: #{msg}\n" }
+        logger
+      end
+
+      DEFAULT_TRACER = lambda do
+        require 'logger'
+        logger = Logger.new(STDERR)
+        logger.progname = 'elastic.tracer'
+        logger.formatter = proc { |severity, datetime, progname, msg| "#{msg}\n" }
+        logger
+      end
+
+      # The default host and port to use if not otherwise specified.
+      #
+      # @since 7.0.0
+      DEFAULT_HOST = 'localhost:9200'.freeze
+
+      # The default port to use if not otherwise specified.
+      #
+      # @since 7.2.0
+      DEFAULT_PORT = 9200
+
+      # Returns the transport object.
+      #
+      # @see Elastic::Transport::Transport::Base
+      # @see Elastic::Transport::Transport::HTTP::Faraday
+      #
+      attr_accessor :transport
+
+      # Create a client connected to an Elastic cluster.
+      #
+      # Specify the URL via arguments or set the `ELASTICSEARCH_URL` environment variable.
+      #
+      # @option arguments [String,Array] :hosts Single host passed as a String or Hash, or multiple hosts
+      #                                         passed as an Array; `host` or `url` keys are also valid
+      #
+      # @option arguments [Boolean] :log   Use the default logger (disabled by default)
+      #
+      # @option arguments [Boolean] :trace Use the default tracer (disabled by default)
+      #
+      # @option arguments [Object] :logger An instance of a Logger-compatible object
+      #
+      # @option arguments [Object] :tracer An instance of a Logger-compatible object
+      #
+      # @option arguments [Number] :resurrect_after After how many seconds a dead connection should be tried again
+      #
+      # @option arguments [Boolean,Number] :reload_connections Reload connections after X requests (false by default)
+      #
+      # @option arguments [Boolean] :randomize_hosts   Shuffle connections on initialization and reload (false by default)
+      #
+      # @option arguments [Integer] :sniffer_timeout   Timeout for reloading connections in seconds (1 by default)
+      #
+      # @option arguments [Boolean,Number] :retry_on_failure   Retry X times when request fails before raising and
+      #                                                        exception (false by default)
+      # @option arguments Array<Number> :retry_on_status Retry when specific status codes are returned
+      #
+      # @option arguments [Boolean] :reload_on_failure Reload connections after failure (false by default)
+      #
+      # @option arguments [Integer] :request_timeout The request timeout to be passed to transport in options
+      #
+      # @option arguments [Symbol] :adapter A specific adapter for Faraday (e.g. `:patron`)
+      #
+      # @option arguments [Hash] :transport_options Options to be passed to the `Faraday::Connection` constructor
+      #
+      # @option arguments [Constant] :transport_class  A specific transport class to use, will be initialized by
+      #                                                the client and passed hosts and all arguments
+      #
+      # @option arguments [Object] :transport A specific transport instance
+      #
+      # @option arguments [Constant] :serializer_class A specific serializer class to use, will be initialized by
+      #                                               the transport and passed the transport instance
+      #
+      # @option arguments [Constant] :selector An instance of selector strategy implemented with
+      #                                        {Elastic::Transport::Transport::Connections::Selector::Base}.
+      #
+      # @option arguments [String] :send_get_body_as Specify the HTTP method to use for GET requests with a body.
+      #                                              (Default: GET)
+      # @option arguments [true, false] :compression Whether to compress requests. Gzip compression will be used.
+      #   The default is false. Responses will automatically be inflated if they are compressed.
+      #   If a custom transport object is used, it must handle the request compression and response inflation.
+      #
+      # @option enable_meta_header [Boolean] :enable_meta_header Enable sending the meta data header to Cloud.
+      #                                                          (Default: true)
+      #
+      # @yield [faraday] Access and configure the `Faraday::Connection` instance directly with a block
+      #
+      def initialize(arguments = {}, &block)
+        @options = arguments.transform_keys(&:to_sym)
+        @arguments = @options
+        @arguments[:logger] ||= @arguments[:log]   ? DEFAULT_LOGGER.call() : nil
+        @arguments[:tracer] ||= @arguments[:trace] ? DEFAULT_TRACER.call() : nil
+        @arguments[:reload_connections] ||= false
+        @arguments[:retry_on_failure]   ||= false
+        @arguments[:reload_on_failure]  ||= false
+        @arguments[:randomize_hosts]    ||= false
+        @arguments[:transport_options]  ||= {}
+        @arguments[:http]               ||= {}
+        @arguments[:enable_meta_header] = arguments.fetch(:enable_meta_header, true)
+        @options[:http]                 ||= {}
+
+        @hosts ||= __extract_hosts(@arguments[:hosts] ||
+                                   @arguments[:host] ||
+                                   @arguments[:url] ||
+                                   @arguments[:urls] ||
+                                   ENV['ELASTICSEARCH_URL'] ||
+                                   DEFAULT_HOST)
+
+        @send_get_body_as = @arguments[:send_get_body_as] || 'GET'
+
+        if @arguments[:request_timeout]
+          @arguments[:transport_options][:request] = { timeout: @arguments[:request_timeout] }
+        end
+
+        if @arguments[:transport]
+          @transport = @arguments[:transport]
+        else
+          @transport_class = @arguments[:transport_class] || DEFAULT_TRANSPORT_CLASS
+          @transport = if @transport_class == Transport::HTTP::Faraday
+                         @arguments[:adapter] ||= __auto_detect_adapter
+                         set_meta_header # from include MetaHeader
+                         @transport_class.new(hosts: @hosts, options: @arguments) do |faraday|
+                           faraday.adapter(@arguments[:adapter])
+                           block&.call faraday
+                         end
+                       else
+                         set_meta_header # from include MetaHeader
+                         @transport_class.new(hosts: @hosts, options: @arguments)
+                       end
+        end
+      end
+
+      # Performs a request through delegation to {#transport}.
+      #
+      def perform_request(method, path, params = {}, body = nil, headers = nil)
+        method = @send_get_body_as if 'GET' == method && body
+        transport.perform_request(method, path, params, body, headers)
+      end
+
+      private
+
+      def add_header(header)
+        headers = @arguments[:transport_options]&.[](:headers) || {}
+        headers.merge!(header)
+        @arguments[:transport_options].merge!(
+          headers: headers
+        )
+      end
+
+      # Normalizes and returns hosts configuration.
+      #
+      # Arrayifies the `hosts_config` argument and extracts `host` and `port` info from strings.
+      # Performs shuffling when the `randomize_hosts` option is set.
+      #
+      # TODO: Refactor, so it's available in Elasticsearch::Transport::Base as well
+      #
+      # @return [Array<Hash>]
+      # @raise  [ArgumentError]
+      #
+      # @api private
+      #
+      def __extract_hosts(hosts_config)
+        hosts = case hosts_config
+                when String
+                  hosts_config.split(',').map { |h| h.strip! || h }
+                when Array
+                  hosts_config
+                when Hash, URI
+                  [hosts_config]
+                else
+                  Array(hosts_config)
+                end
+
+        host_list = hosts.map { |host| __parse_host(host) }
+        @options[:randomize_hosts] ? host_list.shuffle! : host_list
+      end
+
+      def __parse_host(host)
+        host_parts = case host
+                     when String
+                       if host =~ /^[a-z]+\:\/\//
+                         # Construct a new `URI::Generic` directly from the array returned by URI::split.
+                         # This avoids `URI::HTTP` and `URI::HTTPS`, which supply default ports.
+                         uri = URI::Generic.new(*URI.split(host))
+
+                         default_port = uri.scheme == 'https' ? 443 : DEFAULT_PORT
+
+                         { :scheme => uri.scheme,
+                           :user => uri.user,
+                           :password => uri.password,
+                           :host => uri.host,
+                           :path => uri.path,
+                           :port => uri.port || default_port }
+                       else
+                         host, port = host.split(':')
+                         { :host => host,
+                           :port => port }
+                       end
+                     when URI
+                       { :scheme => host.scheme,
+                         :user => host.user,
+                         :password => host.password,
+                         :host => host.host,
+                         :path => host.path,
+                         :port => host.port }
+                     when Hash
+                       host
+                     else
+                       raise ArgumentError, "Please pass host as a String, URI or Hash -- #{host.class} given."
+                     end
+
+        @options[:http][:user] ||= host_parts[:user]
+        @options[:http][:password] ||= host_parts[:password]
+        host_parts[:port] = host_parts[:port].to_i if host_parts[:port]
+        host_parts[:path].chomp!('/') if host_parts[:path]
+        host_parts
+      end
+
+      # Auto-detect the best adapter (HTTP "driver") available, based on libraries
+      # loaded by the user, preferring those with persistent connections
+      # ("keep-alive") by default
+      #
+      # @return [Symbol]
+      #
+      # @api private
+      #
+      def __auto_detect_adapter
+        case
+        when defined?(::Patron)
+          :patron
+        when defined?(::Typhoeus)
+          :typhoeus
+        when defined?(::HTTPClient)
+          :httpclient
+        when defined?(::Net::HTTP::Persistent)
+          :net_http_persistent
+        else
+          ::Faraday.default_adapter
+        end
+      end
+    end
+  end
+end

data/lib/elastic/transport/meta_header.rb

@@ -0,0 +1,135 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require 'base64'
+
+module Elastic
+  module Transport
+    # Methods for the Elastic meta header used by Cloud.
+    # X-Elastic-Client-Meta HTTP header which is used by Elastic Cloud and can be disabled when
+    # instantiating the Client with the :enable_meta_header parameter set to `false`.
+    #
+    module MetaHeader
+      def set_meta_header
+        return if @arguments[:enable_meta_header] == false
+
+        service, version = meta_header_service_version
+
+        meta_headers = {
+          service.to_sym => version,
+          rb: RUBY_VERSION,
+          t: Elastic::Transport::VERSION
+        }
+        meta_headers.merge!(meta_header_engine) if meta_header_engine
+        meta_headers.merge!(meta_header_adapter) if meta_header_adapter
+
+        add_header({ 'x-elastic-client-meta' => meta_headers.map { |k, v| "#{k}=#{v}" }.join(',') })
+      end
+
+      def meta_header_service_version
+        if enterprise_search?
+          Elastic::ENTERPRISE_SERVICE_VERSION
+        elsif elasticsearch?
+          Elastic::ELASTICSEARCH_SERVICE_VERSION
+        elsif defined?(Elasticsearch::VERSION)
+          [:es, client_meta_version(Elasticsearch::VERSION)]
+        else
+          [:es, client_meta_version(Elastic::Transport::VERSION)]
+        end
+      end
+
+      def enterprise_search?
+        defined?(Elastic::ENTERPRISE_SERVICE_VERSION) &&
+          called_from?('enterprise-search-ruby')
+      end
+
+      def elasticsearch?
+        defined?(Elastic::ELASTICSEARCH_SERVICE_VERSION) &&
+          called_from?('elasticsearch')
+      end
+
+      def called_from?(service)
+        !caller.select { |c| c.match?(service) }.empty?
+      end
+
+      # We return the current version if it's a release, but if it's a pre/alpha/beta release we
+      # return <VERSION_NUMBER>p
+      #
+      def client_meta_version(version)
+        regexp = /^([0-9]+\.[0-9]+\.[0-9]+)(\.?[a-z0-9.-]+)?$/
+        match = version.match(regexp)
+        return "#{match[1]}p" if (match[2])
+
+        version
+      end
+
+      def meta_header_engine
+        case RUBY_ENGINE
+        when 'ruby'
+          {}
+        when 'jruby'
+          { jv: ENV_JAVA['java.version'], jr: JRUBY_VERSION }
+        when 'rbx'
+          { rbx: RUBY_VERSION }
+        else
+          { RUBY_ENGINE.to_sym => RUBY_VERSION }
+        end
+      end
+
+      # This function tries to define the version for the Faraday adapter. If it hasn't been loaded
+      # by the time we're calling this method, it's going to report the adapter (if we know it) but
+      # return 0 as the version. It won't report anything when using a custom adapter we don't
+      # identify.
+      #
+      # Returns a Hash<adapter_alias, version>
+      #
+      def meta_header_adapter
+        if @transport_class == Transport::HTTP::Faraday
+          version = '0'
+          adapter_version = case @arguments[:adapter]
+                   when :patron
+                     version = Patron::VERSION if defined?(::Patron::VERSION)
+                     {pt: version}
+                   when :net_http
+                     version = if defined?(Net::HTTP::VERSION)
+                                 Net::HTTP::VERSION
+                               elsif defined?(Net::HTTP::HTTPVersion)
+                                 Net::HTTP::HTTPVersion
+                               end
+                     {nh: version}
+                   when :typhoeus
+                     version = Typhoeus::VERSION if defined?(::Typhoeus::VERSION)
+                     {ty: version}
+                   when :httpclient
+                     version = HTTPClient::VERSION if defined?(HTTPClient::VERSION)
+                     {hc: version}
+                   when :net_http_persistent
+                     version = Net::HTTP::Persistent::VERSION if defined?(Net::HTTP::Persistent::VERSION)
+                     {np: version}
+                   else
+                     {}
+                   end
+          {fd: Faraday::VERSION}.merge(adapter_version)
+        elsif defined?(Transport::HTTP::Curb) && @transport_class == Transport::HTTP::Curb
+          {cl: Curl::CURB_VERSION}
+        elsif defined?(Transport::HTTP::Manticore) && @transport_class == Transport::HTTP::Manticore
+          {mc: Manticore::VERSION}
+        end
+      end
+    end
+  end
+end