elastic-transport 8.0.0 → 8.4.0

data/lib/elastic/transport/client.rb

@@ -15,7 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-require 'base64'
 require 'elastic/transport/meta_header'
 
 module Elastic
@@ -84,7 +83,7 @@ module Elastic
       #
       # @option arguments [Integer] :sniffer_timeout   Timeout for reloading connections in seconds (1 by default)
       #
-      # @option arguments [Boolean,Number] :retry_on_failure   Retry X times when request fails before raising and
+      # @option arguments [Boolean,Number] :retry_on_failure   Retry X times when request fails before raising an
       #                                                        exception (false by default)
       # @option arguments [Number] :delay_on_retry  Delay in milliseconds between each retry (0 by default)
       #
@@ -92,7 +91,8 @@ module Elastic
       #
       # @option arguments [Boolean] :reload_on_failure Reload connections after failure (false by default)
       #
-      # @option arguments [Integer] :request_timeout The request timeout to be passed to transport in options
+      # @option arguments [Integer] :request_timeout The request timeout to be passed to transport in options in seconds 
+      #                                              (the default value is taken from the transport)
       #
       # @option arguments [Symbol] :adapter A specific adapter for Faraday (e.g. `:patron`)
       #
@@ -111,7 +111,7 @@ module Elastic
       #
       # @option arguments [String] :send_get_body_as Specify the HTTP method to use for GET requests with a body.
       #                                              (Default: GET)
-      # @option arguments [true, false] :compression Whether to compress requests. Gzip compression will be used.
+      # @option arguments [Boolean] :compression Whether to compress requests. Gzip compression will be used.
       #   The default is false. Responses will automatically be inflated if they are compressed.
       #   If a custom transport object is used, it must handle the request compression and response inflation.
       #
@@ -122,10 +122,9 @@ module Elastic
       # @yield [faraday] Access and configure the `Faraday::Connection` instance directly with a block
       #
       def initialize(arguments = {}, &block)
-        @options = arguments.transform_keys(&:to_sym)
-        @arguments = @options
-        @arguments[:logger] ||= @arguments[:log]   ? DEFAULT_LOGGER.call() : nil
-        @arguments[:tracer] ||= @arguments[:trace] ? DEFAULT_TRACER.call() : nil
+        @arguments = arguments.transform_keys(&:to_sym)
+        @arguments[:logger] ||= @arguments[:log]   ? DEFAULT_LOGGER.call : nil
+        @arguments[:tracer] ||= @arguments[:trace] ? DEFAULT_TRACER.call : nil
         @arguments[:reload_connections] ||= false
         @arguments[:retry_on_failure]   ||= false
         @arguments[:delay_on_retry]     ||= 0
@@ -134,15 +133,8 @@ module Elastic
         @arguments[:transport_options]  ||= {}
         @arguments[:http]               ||= {}
         @arguments[:enable_meta_header] = arguments.fetch(:enable_meta_header, true)
-        @options[:http]                 ||= {}
-
-        @hosts ||= __extract_hosts(@arguments[:hosts] ||
-                                   @arguments[:host] ||
-                                   @arguments[:url] ||
-                                   @arguments[:urls] ||
-                                   ENV['ELASTICSEARCH_URL'] ||
-                                   DEFAULT_HOST)
 
+        @hosts ||= extract_hosts
         @send_get_body_as = @arguments[:send_get_body_as] || 'GET'
         @ca_fingerprint = @arguments.delete(:ca_fingerprint)
 
@@ -155,7 +147,7 @@ module Elastic
         else
           @transport_class = @arguments[:transport_class] || DEFAULT_TRANSPORT_CLASS
           @transport = if @transport_class == Transport::HTTP::Faraday
-                         @arguments[:adapter] ||= __auto_detect_adapter
+                         @arguments[:adapter] ||= auto_detect_adapter
                          set_meta_header # from include MetaHeader
                          @transport_class.new(hosts: @hosts, options: @arguments) do |faraday|
                            faraday.adapter(@arguments[:adapter])
@@ -166,14 +158,39 @@ module Elastic
                          @transport_class.new(hosts: @hosts, options: @arguments)
                        end
         end
+
+        if defined?(::OpenTelemetry) && ENV[OpenTelemetry::ENV_VARIABLE_ENABLED] != 'false'
+          @otel = OpenTelemetry.new(@arguments)
+        end
       end
 
       # Performs a request through delegation to {#transport}.
       #
-      def perform_request(method, path, params = {}, body = nil, headers = nil)
-        method = @send_get_body_as if 'GET' == method && body
+      def perform_request(method, path, params = {}, body = nil, headers = nil, opts = {})
+        method = @send_get_body_as if method == 'GET' && body
         validate_ca_fingerprints if @ca_fingerprint
-        transport.perform_request(method, path, params, body, headers)
+        if @otel
+          # If no endpoint is specified in the opts, use the HTTP method name
+          span_name = opts[:endpoint] || method
+          @otel.tracer.in_span(span_name) do |span|
+            span['http.request.method'] = method
+            span['db.system'] = 'elasticsearch'
+            opts[:defined_params]&.each do |k, v|
+              if v.respond_to?(:join)
+                span["db.elasticsearch.path_parts.#{k}"] = v.join(',')
+              else
+                span["db.elasticsearch.path_parts.#{k}"] = v
+              end
+            end
+            if body_as_json = @otel.process_body(body, opts[:endpoint])
+              span['db.statement'] = body_as_json
+            end
+            span['db.operation'] = opts[:endpoint] if opts[:endpoint]
+            transport.perform_request(method, path, params || {}, body, headers)
+          end
+        else
+          transport.perform_request(method, path, params || {}, body, headers)
+        end
       end
 
       private
@@ -211,14 +228,6 @@ module Elastic
         )
       end
 
-      def add_header(header)
-        headers = @arguments[:transport_options]&.[](:headers) || {}
-        headers.merge!(header)
-        @arguments[:transport_options].merge!(
-          headers: headers
-        )
-      end
-
       # Normalizes and returns hosts configuration.
       #
       # Arrayifies the `hosts_config` argument and extracts `host` and `port` info from strings.
@@ -231,7 +240,7 @@ module Elastic
       #
       # @api private
       #
-      def __extract_hosts(hosts_config)
+      def extract_hosts
         hosts = case hosts_config
                 when String
                   hosts_config.split(',').map { |h| h.strip! || h }
@@ -242,12 +251,20 @@ module Elastic
                 else
                   Array(hosts_config)
                 end
+        host_list = hosts.map { |host| parse_host(host) }
+        @arguments[:randomize_hosts] ? host_list.shuffle! : host_list
+      end
 
-        host_list = hosts.map { |host| __parse_host(host) }
-        @options[:randomize_hosts] ? host_list.shuffle! : host_list
+      def hosts_config
+        @arguments[:hosts] ||
+          @arguments[:host] ||
+          @arguments[:url] ||
+          @arguments[:urls] ||
+          ENV['ELASTICSEARCH_URL'] ||
+          DEFAULT_HOST
       end
 
-      def __parse_host(host)
+      def parse_host(host)
         host_parts = case host
                      when String
                        if host =~ /^[a-z]+\:\/\//
@@ -281,8 +298,8 @@ module Elastic
                        raise ArgumentError, "Please pass host as a String, URI or Hash -- #{host.class} given."
                      end
 
-        @options[:http][:user] ||= host_parts[:user]
-        @options[:http][:password] ||= host_parts[:password]
+        @arguments[:http][:user] ||= host_parts[:user]
+        @arguments[:http][:password] ||= host_parts[:password]
         host_parts[:port] = host_parts[:port].to_i if host_parts[:port]
         host_parts[:path].chomp!('/') if host_parts[:path]
         host_parts
@@ -291,24 +308,28 @@ module Elastic
       # Auto-detect the best adapter (HTTP "driver") available, based on libraries
       # loaded by the user, preferring those with persistent connections
       # ("keep-alive") by default
+      # Check adapters based on the usage of Faraday 1 or 2. Faraday should be defined here
+      # since this is only called when transport class is Transport::HTTP::Faraday
       #
       # @return [Symbol]
       #
       # @api private
       #
-      def __auto_detect_adapter
-        case
-        when defined?(::Patron)
-          :patron
-        when defined?(::Typhoeus)
-          :typhoeus
-        when defined?(::HTTPClient)
-          :httpclient
-        when defined?(::Net::HTTP::Persistent)
-          :net_http_persistent
+      def auto_detect_adapter
+        if Gem::Version.new(Faraday::VERSION) >= Gem::Version.new(2)
+          return :patron if defined?(Faraday::Adapter::Patron)
+          return :typhoeus if defined?(Faraday::Adapter::Typhoeus)
+          return :httpclient if defined?(Faraday::Adapter::HTTPClient)
+          return :net_http_persistent if defined?(Faraday::Adapter::NetHttpPersistent)
+          return :excon if defined?(Faraday::Adapter::Excon)
+          return :async_http if defined?(Async::HTTP::Faraday)
         else
-          ::Faraday.default_adapter
+          return :patron if defined?(::Patron)
+          return :typhoeus if defined?(::Typhoeus)
+          return :httpclient if defined?(::HTTPClient)
+          return :net_http_persistent if defined?(::Net::HTTP::Persistent)
         end
+        ::Faraday.default_adapter
       end
     end
   end

data/lib/elastic/transport/meta_header.rb

@@ -15,8 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-require 'base64'
-
 module Elastic
   module Transport
     # Methods for the Elastic meta header used by Cloud.
@@ -43,12 +41,18 @@ module Elastic
       def meta_header_service_version
         if enterprise_search?
           Elastic::ENTERPRISE_SERVICE_VERSION
+        elsif serverless?
+          if defined?(Elastic::ES_SERVERLESS_SERVICE_VERSION)
+            Elastic::ES_SERVERLESS_SERVICE_VERSION
+          else
+            Elastic::ELASTICSEARCH_SERVICE_VERSION
+          end
         elsif elasticsearch?
           Elastic::ELASTICSEARCH_SERVICE_VERSION
         elsif defined?(Elasticsearch::VERSION)
           [:es, client_meta_version(Elasticsearch::VERSION)]
         else
-          [:es, client_meta_version(Elastic::Transport::VERSION)]
+          [:et, client_meta_version(Elastic::Transport::VERSION)]
         end
       end
 
@@ -62,6 +66,11 @@ module Elastic
           called_from?('elasticsearch')
       end
 
+      def serverless?
+        defined?(ElasticsearchServerless::CLIENT_VERSION) &&
+          called_from?('elasticsearch-serverless')
+      end
+
       def called_from?(service)
         !caller.select { |c| c.match?(service) }.empty?
       end
@@ -72,7 +81,7 @@ module Elastic
       def client_meta_version(version)
         regexp = /^([0-9]+\.[0-9]+\.[0-9]+)(\.?[a-z0-9.-]+)?$/
         match = version.match(regexp)
-        return "#{match[1]}p" if (match[2])
+        return "#{match[1]}p" if match[2]
 
         version
       end
@@ -103,31 +112,31 @@ module Elastic
           adapter_version = case @arguments[:adapter]
                    when :patron
                      version = Patron::VERSION if defined?(::Patron::VERSION)
-                     {pt: version}
+                     { pt: version }
                    when :net_http
                      version = if defined?(Net::HTTP::VERSION)
                                  Net::HTTP::VERSION
                                elsif defined?(Net::HTTP::HTTPVersion)
                                  Net::HTTP::HTTPVersion
                                end
-                     {nh: version}
+                     { nh: version }
                    when :typhoeus
                      version = Typhoeus::VERSION if defined?(::Typhoeus::VERSION)
-                     {ty: version}
+                     { ty: version }
                    when :httpclient
                      version = HTTPClient::VERSION if defined?(HTTPClient::VERSION)
-                     {hc: version}
+                     { hc: version }
                    when :net_http_persistent
                      version = Net::HTTP::Persistent::VERSION if defined?(Net::HTTP::Persistent::VERSION)
-                     {np: version}
+                     { np: version }
                    else
                      {}
                    end
-          {fd: Faraday::VERSION}.merge(adapter_version)
+          { fd: Faraday::VERSION }.merge(adapter_version)
         elsif defined?(Transport::HTTP::Curb) && @transport_class == Transport::HTTP::Curb
-          {cl: Curl::CURB_VERSION}
+          { cl: Curl::CURB_VERSION }
         elsif defined?(Transport::HTTP::Manticore) && @transport_class == Transport::HTTP::Manticore
-          {mc: Manticore::VERSION}
+          { mc: Manticore::VERSION }
         end
       end
     end

data/lib/elastic/transport/opentelemetry.rb

@@ -0,0 +1,166 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Elastic
+  module Transport
+    # Wrapper object for Open Telemetry objects, associated config and functionality.
+    #
+    # @api private
+    class OpenTelemetry
+      OTEL_TRACER_NAME = 'elasticsearch-api'.freeze
+      # Valid values for the enabled config are 'true' and 'false'. Default is 'true'.
+      ENV_VARIABLE_ENABLED = 'OTEL_RUBY_INSTRUMENTATION_ELASTICSEARCH_ENABLED'.freeze
+      # Describes how to handle search queries in the request body when assigned to
+      # a span attribute.
+      # Valid values are 'raw', 'omit', 'sanitize'. Default is 'omit'.
+      ENV_VARIABLE_BODY_STRATEGY = 'OTEL_RUBY_INSTRUMENTATION_ELASTICSEARCH_CAPTURE_SEARCH_QUERY'.freeze
+      ENV_VARIABLE_DEPRECATED_BODY_STRATEGY = 'OTEL_INSTRUMENTATION_ELASTICSEARCH_CAPTURE_SEARCH_QUERY'.freeze
+      DEFAULT_BODY_STRATEGY = 'omit'.freeze
+      # A string list of keys whose values are redacted. This is only relevant if the body strategy is
+      # 'sanitize'. For example, a config 'sensitive-key,other-key' will redact the values at
+      # 'sensitive-key' and 'other-key' in addition to the default keys.
+      ENV_VARIABLE_BODY_SANITIZE_KEYS = 'OTEL_RUBY_INSTRUMENTATION_ELASTICSEARCH_SEARCH_QUERY_SANITIZE_KEYS'.freeze
+
+      # A list of the Elasticsearch endpoints that qualify as "search" endpoints. The search query in
+      # the request body may be captured for these endpoints, depending on the body capture strategy.
+      SEARCH_ENDPOINTS = Set[
+        'search',
+        'async_search.submit',
+        'msearch',
+        'eql.search',
+        'terms_enum',
+        'search_template',
+        'msearch_template',
+        'render_search_template',
+      ]
+
+      # Initialize the Open Telemetry wrapper object. Takes the options originally passed to
+      # Client#initialize.
+      def initialize(opts)
+        @tracer = tracer_provider(opts).tracer(OTEL_TRACER_NAME, Elastic::Transport::VERSION)
+        @body_strategy = ENV[ENV_VARIABLE_DEPRECATED_BODY_STRATEGY] ||
+                         ENV[ENV_VARIABLE_BODY_STRATEGY] ||
+                         DEFAULT_BODY_STRATEGY
+        @sanitize_keys = ENV[ENV_VARIABLE_BODY_SANITIZE_KEYS]&.split(',')&.collect! do |pattern|
+          Regexp.new(pattern.gsub('*', '.*'))
+        end
+      end
+      attr_accessor :tracer
+
+      def tracer_provider(opts)
+        opts[:opentelemetry_tracer_provider] || ::OpenTelemetry.tracer_provider
+      end
+
+      # Process the request body. Applies the body strategy, which can be one of the following:
+      # 'omit' (DEFAULT_BODY_STRATEGY): return nil
+      # 'sanitize': redact values at the default list of keys + any additional keys provided in
+      # the OTEL_RUBY_INSTRUMENTATION_ELASTICSEARCH_SEARCH_QUERY_SANITIZE_KEYS env variable.
+      # 'raw': return the original body, unchanged
+      def process_body(body, endpoint)
+        return if @body_strategy == DEFAULT_BODY_STRATEGY || !search_endpoint?(endpoint)
+
+        if @body_strategy == 'sanitize'
+          Sanitizer.sanitize(body, @sanitize_keys).to_json
+        elsif @body_strategy == 'raw'
+          body.is_a?(String) ? body : body.to_json
+        end
+      end
+
+      def search_endpoint?(endpoint)
+        SEARCH_ENDPOINTS.include?(endpoint)
+      end
+
+      # Replaces values in a hash with 'REDACTED', given a set of keys to match on.
+      class Sanitizer
+        class << self
+          FILTERED = 'REDACTED'.freeze
+          DEFAULT_KEY_PATTERNS =
+            %w[password passwd pwd secret *key *token* *session* *credit* *card* *auth* set-cookie].map! do |p|
+            Regexp.new(p.gsub('*', '.*'))
+          end
+
+          def sanitize(body, key_patterns = [])
+            patterns = DEFAULT_KEY_PATTERNS
+            patterns += key_patterns if key_patterns
+            sanitize!(DeepDup.dup(body), patterns)
+          end
+
+          private
+
+          def sanitize!(obj, key_patterns)
+            return obj unless obj.is_a?(Hash)
+
+            obj.each_pair do |k, v|
+              if filter_key?(key_patterns, k)
+                obj[k] = FILTERED
+              elsif v.is_a?(Hash)
+                sanitize!(v, key_patterns)
+              else
+                next
+              end
+            end
+          end
+
+          def filter_key?(key_patterns, key)
+            key_patterns.any? { |regex| regex.match(key) }
+          end
+        end
+      end
+
+      # Makes a deep copy of an Array or Hash
+      # NB: Not guaranteed to work well with complex objects, only simple Hash,
+      # Array, String, Number, etc.
+      class DeepDup
+        def initialize(obj)
+          @obj = obj
+        end
+
+        def dup
+          deep_dup(@obj)
+        end
+
+        def self.dup(obj)
+          new(obj).dup
+        end
+
+        private
+
+        def deep_dup(obj)
+          case obj
+          when Hash then hash(obj)
+          when Array then array(obj)
+          else obj.dup
+          end
+        end
+
+        def array(arr)
+          arr.map { |obj| deep_dup(obj) }
+        end
+
+        def hash(hsh)
+          result = hsh.dup
+
+          hsh.each_pair do |key, value|
+            result[key] = deep_dup(value)
+          end
+
+          result
+        end
+      end
+    end
+  end
+end