datadog 2.0.0.beta1

data/lib/datadog/tracing/contrib/elasticsearch/ext.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Tracing
+    module Contrib
+      module Elasticsearch
+        # Elasticsearch integration constants
+        # @public_api Changing resource names, tag names, or environment variables creates breaking changes.
+        module Ext
+          ENV_ENABLED = 'DD_TRACE_ELASTICSEARCH_ENABLED'
+          ENV_SERVICE_NAME = 'DD_TRACE_ELASTICSEARCH_SERVICE_NAME'
+          ENV_PEER_SERVICE = 'DD_TRACE_ELASTICSEARCH_PEER_SERVICE'
+          # @!visibility private
+          ENV_ANALYTICS_ENABLED = 'DD_TRACE_ELASTICSEARCH_ANALYTICS_ENABLED'
+          ENV_ANALYTICS_SAMPLE_RATE = 'DD_TRACE_ELASTICSEARCH_ANALYTICS_SAMPLE_RATE'
+          DEFAULT_PEER_SERVICE_NAME = 'elasticsearch'
+          SPAN_QUERY = 'elasticsearch.query'
+          SPAN_TYPE_QUERY = 'elasticsearch'
+          TAG_BODY = 'elasticsearch.body'
+          TAG_METHOD = 'elasticsearch.method'
+          TAG_PARAMS = 'elasticsearch.params'
+          TAG_URL = 'elasticsearch.url'
+          TAG_COMPONENT = 'elasticsearch'
+          TAG_OPERATION_QUERY = 'query'
+          TAG_SYSTEM = 'elasticsearch'
+          PEER_SERVICE_SOURCES = Array[
+            Tracing::Metadata::Ext::TAG_PEER_HOSTNAME,
+            Tracing::Metadata::Ext::NET::TAG_DESTINATION_NAME,
+            Tracing::Metadata::Ext::NET::TAG_TARGET_HOST,].freeze
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/contrib/elasticsearch/integration.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative '../integration'
+require_relative 'configuration/settings'
+require_relative 'patcher'
+
+module Datadog
+  module Tracing
+    module Contrib
+      module Elasticsearch
+        # Description of Elasticsearch integration
+        class Integration
+          include Contrib::Integration
+
+          MINIMUM_VERSION = Gem::Version.new('1.0.0')
+
+          # @public_api Changing the integration name or integration options can cause breaking changes
+          register_as :elasticsearch, auto_patch: true
+
+          def self.version
+            # elastic-transport gem for version >= 8.0.0
+            # elasticsearch-transport gem for version < 8.0.0
+            Gem.loaded_specs['elastic-transport'] \
+              && Gem.loaded_specs['elastic-transport'].version || \
+              Gem.loaded_specs['elasticsearch-transport'] \
+                && Gem.loaded_specs['elasticsearch-transport'].version
+          end
+
+          def self.loaded?
+            # Elastic::Transport gem for version >= 8.0.0
+            # Elasticsearch::Transport gem for version < 8.0.0
+            !defined?(::Elastic::Transport).nil? || !defined?(::Elasticsearch::Transport).nil?
+          end
+
+          def self.compatible?
+            super && version >= MINIMUM_VERSION
+          end
+
+          def new_configuration
+            Configuration::Settings.new
+          end
+
+          def patcher
+            Patcher
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/contrib/elasticsearch/patcher.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+require_relative '../../metadata/ext'
+require_relative '../analytics'
+require_relative 'ext'
+require_relative '../ext'
+require_relative '../integration'
+require_relative '../patcher'
+
+module Datadog
+  module Tracing
+    module Contrib
+      module Elasticsearch
+        # Patcher enables patching of 'elasticsearch' module.
+        module Patcher
+          include Contrib::Patcher
+
+          module_function
+
+          def target_version
+            Integration.version
+          end
+
+          def patch
+            require 'uri'
+            require 'json'
+            require_relative 'quantize'
+
+            transport_module::Client.prepend(DatadogPin)
+            transport_module::Client.prepend(Client)
+          end
+
+          SELF_DEPRECATION_ONLY_ONCE = Core::Utils::OnlyOnce.new
+
+          # Patches Elasticsearch::Transport::Client module
+          module Client
+            # rubocop:disable Metrics/MethodLength
+            # rubocop:disable Metrics/AbcSize
+            def perform_request(*args)
+              # `Client#transport` is the most convenient object both for this integration and the library
+              # as users have shared access to it across all `elasticsearch` versions.
+              service ||= Datadog.configuration_for(transport, :service_name) || datadog_configuration[:service_name]
+
+              method = args[0]
+              path = args[1]
+              params = args[2]
+              body = args[3]
+              full_url = URI.parse(path)
+              url = full_url.path
+              response = nil
+
+              Tracing.trace(Datadog::Tracing::Contrib::Elasticsearch::Ext::SPAN_QUERY, service: service) do |span|
+                begin
+                  connection = transport.connections.first
+                  host = connection.host[:host] if connection
+                  port = connection.host[:port] if connection
+
+                  if datadog_configuration[:peer_service]
+                    span.set_tag(
+                      Tracing::Metadata::Ext::TAG_PEER_SERVICE,
+                      datadog_configuration[:peer_service]
+                    )
+                  end
+
+                  # Tag original global service name if not used
+                  if span.service != Datadog.configuration.service
+                    span.set_tag(Tracing::Contrib::Ext::Metadata::TAG_BASE_SERVICE, Datadog.configuration.service)
+                  end
+
+                  span.type = Datadog::Tracing::Contrib::Elasticsearch::Ext::SPAN_TYPE_QUERY
+
+                  span.set_tag(Tracing::Metadata::Ext::TAG_COMPONENT, Ext::TAG_COMPONENT)
+                  span.set_tag(Tracing::Metadata::Ext::TAG_OPERATION, Ext::TAG_OPERATION_QUERY)
+                  span.set_tag(Tracing::Metadata::Ext::TAG_KIND, Tracing::Metadata::Ext::SpanKind::TAG_CLIENT)
+
+                  span.set_tag(Contrib::Ext::DB::TAG_SYSTEM, Ext::TAG_SYSTEM)
+
+                  span.set_tag(Tracing::Metadata::Ext::TAG_PEER_HOSTNAME, host) if host
+
+                  # Set analytics sample rate
+                  if Contrib::Analytics.enabled?(datadog_configuration[:analytics_enabled])
+                    Contrib::Analytics.set_sample_rate(span, datadog_configuration[:analytics_sample_rate])
+                  end
+
+                  span.set_tag(Datadog::Tracing::Contrib::Elasticsearch::Ext::TAG_METHOD, method)
+                  tag_params(params, span)
+                  tag_body(body, span)
+                  span.set_tag(Datadog::Tracing::Contrib::Elasticsearch::Ext::TAG_URL, url)
+                  span.set_tag(Tracing::Metadata::Ext::NET::TAG_TARGET_HOST, host) if host
+                  span.set_tag(Tracing::Metadata::Ext::NET::TAG_TARGET_PORT, port) if port
+
+                  quantized_url = Datadog::Tracing::Contrib::Elasticsearch::Quantize.format_url(url)
+                  span.resource = "#{method} #{quantized_url}"
+                  Contrib::SpanAttributeSchema.set_peer_service!(span, Ext::PEER_SERVICE_SOURCES)
+                rescue StandardError => e
+                  Datadog.logger.error(e.message)
+                ensure
+                  # the call is still executed
+                  response = super
+                  span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_STATUS_CODE, response.status)
+                end
+              end
+              response
+            end
+
+            def tag_params(params, span)
+              return unless params
+
+              params = JSON.generate(params) unless params.is_a?(String)
+              span.set_tag(Datadog::Tracing::Contrib::Elasticsearch::Ext::TAG_PARAMS, params)
+            end
+
+            def tag_body(body, span)
+              return unless body
+
+              body = JSON.generate(body) unless body.is_a?(String)
+              quantize_options = datadog_configuration[:quantize]
+              quantized_body = Datadog::Tracing::Contrib::Elasticsearch::Quantize.format_body(
+                body,
+                quantize_options
+              )
+              span.set_tag(Datadog::Tracing::Contrib::Elasticsearch::Ext::TAG_BODY, quantized_body)
+            end
+
+            def datadog_configuration
+              Datadog.configuration.tracing[:elasticsearch]
+            end
+          end
+          # rubocop:enable Metrics/MethodLength
+          # rubocop:enable Metrics/AbcSize
+
+          # Patch to support both `elasticsearch` and `elastic-transport` versions
+          module DatadogPin
+            def datadog_pin=(pin)
+              pin.onto(pin_candidate)
+            end
+
+            def datadog_pin
+              Datadog.configuration_for(pin_candidate)
+            end
+
+            def pin_candidate(candidate = self)
+              if candidate.respond_to?(:transport)
+                pin_candidate(candidate.transport)
+              else
+                candidate
+              end
+            end
+          end
+
+          # `Elasticsearch` namespace renamed to `Elastic` in version 8.0.0 of the transport gem:
+          # @see https://github.com/elastic/elastic-transport-ruby/commit/ef804cbbd284f2a82d825221f87124f8b5ff823c
+          def transport_module
+            if Integration.version >= Gem::Version.new('8.0.0')
+              ::Elastic::Transport
+            else
+              ::Elasticsearch::Transport
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/contrib/elasticsearch/quantize.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require_relative '../utils/quantization/hash'
+
+module Datadog
+  module Tracing
+    module Contrib
+      module Elasticsearch
+        # Quantize contains ES-specific resource quantization tools.
+        module Quantize
+          PLACEHOLDER = '?'
+          ID_PLACEHOLDER = '\1?'
+          EXCLUDE_KEYS = [].freeze
+          SHOW_KEYS = [:_index, :_type, :_id].freeze
+          DEFAULT_OPTIONS = {
+            exclude: EXCLUDE_KEYS,
+            show: SHOW_KEYS,
+            placeholder: PLACEHOLDER
+          }.freeze
+
+          module_function
+
+          def format_url(url)
+            sanitize_fragment_with_id(url)
+              .gsub(/(?:\d+)/, PLACEHOLDER)
+          end
+
+          def format_body(body, options = {})
+            format_body!(body, options)
+          rescue StandardError
+            options[:placeholder] || PLACEHOLDER
+          end
+
+          def format_body!(body, options = {})
+            options = merge_options(DEFAULT_OPTIONS, options)
+
+            # Determine if bulk query or not, based on content
+            statements = body.end_with?("\n") ? body.split("\n") : [body]
+
+            # Parse each statement and quantize them.
+            statements.collect do |string|
+              reserialize_json(string, options[:placeholder]) do |obj|
+                Contrib::Utils::Quantization::Hash.format(obj, options)
+              end
+            end.join("\n")
+          end
+
+          def merge_options(original, additional)
+            {}.tap do |options|
+              # Show
+              # If either is :all, value becomes :all
+              options[:show] = if original[:show] == :all || additional[:show] == :all
+                                 :all
+                               else
+                                 (original[:show] || []).dup.concat(additional[:show] || []).uniq
+                               end
+
+              # Exclude
+              options[:exclude] = (original[:exclude] || []).dup.concat(additional[:exclude] || []).uniq
+            end
+          end
+
+          # Parses a JSON object from a string, passes its value
+          # to the block provided, and dumps its result back to JSON.
+          # If JSON parsing fails, it prints fail_value.
+          def reserialize_json(string, fail_value = PLACEHOLDER)
+            return string unless block_given?
+
+            begin
+              JSON.dump(yield(JSON.parse(string)))
+            rescue JSON::ParserError
+              # If it can't parse/dump, don't raise an error.
+              fail_value
+            end
+          end
+
+          # Sanitizes URL fragment by changing it to ? whenever a number is detected
+          # This is meant as simple heuristic that attempts to detect if particular fragment
+          # represents document Id. This is meant to reduce the cardinality in most frequent cases.
+          def sanitize_fragment_with_id(url)
+            url.gsub(%r{^(/?[^/]*/[^/]*/)(?:[^?/\d]*\d+[^?/]*)}, ID_PLACEHOLDER)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/contrib/ethon/configuration/settings.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require_relative '../../configuration/settings'
+require_relative '../ext'
+
+module Datadog
+  module Tracing
+    module Contrib
+      module Ethon
+        module Configuration
+          # Custom settings for the Ethon integration
+          # @public_api
+          class Settings < Contrib::Configuration::Settings
+            option :enabled do |o|
+              o.type :bool
+              o.env Ext::ENV_ENABLED
+              o.default true
+            end
+
+            # @!visibility private
+            option :analytics_enabled do |o|
+              o.type :bool
+              o.env Ext::ENV_ANALYTICS_ENABLED
+              o.default false
+            end
+
+            option :analytics_sample_rate do |o|
+              o.type :float
+              o.env Ext::ENV_ANALYTICS_SAMPLE_RATE
+              o.default 1.0
+            end
+
+            option :distributed_tracing, default: true, type: :bool
+
+            option :split_by_domain, default: false, type: :bool
+
+            option :service_name do |o|
+              o.type :string, nilable: true
+              o.default do
+                Contrib::SpanAttributeSchema.fetch_service_name(
+                  Ext::ENV_SERVICE_NAME,
+                  Ext::DEFAULT_PEER_SERVICE_NAME
+                )
+              end
+            end
+
+            option :peer_service do |o|
+              o.type :string, nilable: true
+              o.env Ext::ENV_PEER_SERVICE
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/contrib/ethon/easy_patch.rb

@@ -0,0 +1,223 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+require_relative '../../../core/utils/hash'
+require_relative '../../metadata/ext'
+require_relative '../http'
+require_relative 'ext'
+require_relative '../http_annotation_helper'
+
+module Datadog
+  module Tracing
+    module Contrib
+      module Ethon
+        # Ethon EasyPatch
+        module EasyPatch
+          def self.included(base)
+            base.prepend(InstanceMethods)
+          end
+
+          # InstanceMethods - implementing instrumentation
+          module InstanceMethods
+            include Contrib::HttpAnnotationHelper
+
+            def http_request(url, action_name, options = {})
+              load_datadog_configuration_for(url)
+              return super unless Tracing.enabled?
+
+              # It's tricky to get HTTP method from libcurl
+              @datadog_method = action_name.to_s.upcase
+              super
+            end
+
+            def headers=(headers)
+              # Store headers to call this method again when span is ready
+              @datadog_original_headers = headers
+              super
+            end
+
+            def perform
+              load_datadog_configuration_for(url)
+              return super unless Tracing.enabled?
+
+              datadog_before_request
+              super
+            end
+
+            def complete
+              return super unless Tracing.enabled?
+
+              begin
+                response_options = mirror.options
+                response_code = (response_options[:response_code] || response_options[:code]).to_i
+                if response_code.zero?
+                  return_code = response_options[:return_code]
+                  message = return_code ? ::Ethon::Curl.easy_strerror(return_code) : 'unknown reason'
+                  set_span_error_message("Request has failed: #{message}")
+                else
+                  @datadog_span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_STATUS_CODE, response_code)
+                  if Tracing::Metadata::Ext::HTTP::ERROR_RANGE.cover?(response_code)
+                    set_span_error_message("Request has failed with HTTP error: #{response_code}")
+                  end
+                end
+
+                @datadog_span.set_tags(
+                  Datadog.configuration.tracing.header_tags.response_tags(
+                    Core::Utils::Hash::CaseInsensitiveWrapper.new(parse_response_headers)
+                  )
+                )
+              ensure
+                @datadog_span.finish
+                @datadog_span = nil
+              end
+              super
+            end
+
+            def reset
+              super
+            ensure
+              @datadog_span = nil
+              @datadog_method = nil
+              @datadog_original_headers = nil
+              @datadog_configuration = nil
+            end
+
+            # Starts or retrieves the already started Easy request span.
+            #
+            # When tracing in Multi request context, child spans for each Easy
+            # request are created early, and then finished as their HTTP response
+            # becomes available. Because many Easy requests are open at the same time,
+            # many Datadog::Tracing::Spans are also open at the same time. To avoid each separate
+            # Easy request becoming the parented to the previous open request we set
+            # the +parent_span+ parameter with the parent Multi span. This correctly
+            # assigns all open Easy spans to the currently executing Multi context.
+            #
+            # @param [Datadog::Tracing::Span] continue_from the Multi span, if executing in a Multi context.
+            def datadog_before_request(continue_from: nil)
+              load_datadog_configuration_for(url)
+
+              trace_options = continue_from ? { continue_from: continue_from } : {}
+              uri = try_parse_uri
+
+              @datadog_span = Tracing.trace(
+                Ext::SPAN_REQUEST,
+                service: uri ? service_name(uri.host, datadog_configuration) : datadog_configuration[:service_name],
+                type: Tracing::Metadata::Ext::HTTP::TYPE_OUTBOUND,
+                **trace_options
+              )
+              datadog_trace = Tracing.active_trace
+
+              datadog_tag_request
+
+              if datadog_configuration[:distributed_tracing]
+                @datadog_original_headers ||= {}
+                Contrib::HTTP.inject(datadog_trace, @datadog_original_headers)
+                self.headers = @datadog_original_headers
+              end
+            end
+
+            def datadog_span_started?
+              instance_variable_defined?(:@datadog_span) && !@datadog_span.nil?
+            end
+
+            private
+
+            attr_reader :datadog_configuration
+
+            def datadog_tag_request
+              span = @datadog_span
+              method = Ext::NOT_APPLICABLE_METHOD
+              method = @datadog_method.to_s if instance_variable_defined?(:@datadog_method) && !@datadog_method.nil?
+              span.resource = method
+
+              if datadog_configuration[:peer_service]
+                span.set_tag(
+                  Tracing::Metadata::Ext::TAG_PEER_SERVICE,
+                  datadog_configuration[:peer_service]
+                )
+              end
+
+              # Tag original global service name if not used
+              if span.service != Datadog.configuration.service
+                span.set_tag(Tracing::Contrib::Ext::Metadata::TAG_BASE_SERVICE, Datadog.configuration.service)
+              end
+
+              # Set analytics sample rate
+              Contrib::Analytics.set_sample_rate(span, analytics_sample_rate) if analytics_enabled?
+
+              span.set_tag(Tracing::Metadata::Ext::TAG_COMPONENT, Ext::TAG_COMPONENT)
+              span.set_tag(Tracing::Metadata::Ext::TAG_OPERATION, Ext::TAG_OPERATION_REQUEST)
+
+              span.set_tag(Tracing::Metadata::Ext::TAG_KIND, Tracing::Metadata::Ext::SpanKind::TAG_CLIENT)
+
+              uri = try_parse_uri
+              return unless uri
+
+              span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_URL, uri.path)
+              span.set_tag(Tracing::Metadata::Ext::HTTP::TAG_METHOD, method)
+              span.set_tag(Tracing::Metadata::Ext::NET::TAG_TARGET_HOST, uri.host)
+              span.set_tag(Tracing::Metadata::Ext::NET::TAG_TARGET_PORT, uri.port)
+              span.set_tag(Tracing::Metadata::Ext::TAG_PEER_HOSTNAME, uri.host)
+
+              if @datadog_original_headers
+                span.set_tags(
+                  Datadog.configuration.tracing.header_tags.request_tags(
+                    Core::Utils::Hash::CaseInsensitiveWrapper.new(@datadog_original_headers)
+                  )
+                )
+              end
+
+              Contrib::SpanAttributeSchema.set_peer_service!(span, Ext::PEER_SERVICE_SOURCES)
+            end
+
+            def set_span_error_message(message)
+              # Sets span error from message, in case there is no exception available
+              @datadog_span.status = Tracing::Metadata::Ext::Errors::STATUS
+              @datadog_span.set_tag(Tracing::Metadata::Ext::Errors::TAG_MSG, message)
+            end
+
+            # rubocop:disable Lint/SuppressedException
+            def try_parse_uri
+              URI.parse(url)
+            rescue URI::InvalidURIError
+            end
+            # rubocop:enable Lint/SuppressedException
+
+            def load_datadog_configuration_for(host = :default)
+              @datadog_configuration = Datadog.configuration.tracing[:ethon, host]
+            end
+
+            def analytics_enabled?
+              Contrib::Analytics.enabled?(datadog_configuration[:analytics_enabled])
+            end
+
+            def analytics_sample_rate
+              datadog_configuration[:analytics_sample_rate]
+            end
+
+            # `#response_headers` returns a "\n" concatenated String containing:
+            # * The HTTP Status-Line.
+            # * The response headers.
+            # * A trailing "\n".
+            #
+            # This method extracts only the headers from it.
+            def parse_response_headers
+              return {} if response_headers.empty?
+
+              lines = response_headers.split("\n")
+
+              lines = lines[1..(lines.size - 1)] # Remove Status-Line and trailing whitespace.
+
+              # Find only well-behaved HTTP headers.
+              lines.map do |line|
+                header = line.split(':', 2)
+                header.size != 2 ? nil : header
+              end.compact.to_h
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/contrib/ethon/ext.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Tracing
+    module Contrib
+      module Ethon
+        # Ethon integration constants
+        # @public_api Changing resource names, tag names, or environment variables creates breaking changes.
+        module Ext
+          ENV_ENABLED = 'DD_TRACE_ETHON_ENABLED'
+          ENV_SERVICE_NAME = 'DD_TRACE_ETHON_SERVICE_NAME'
+          ENV_PEER_SERVICE = 'DD_TRACE_ETHON_PEER_SERVICE'
+
+          # @!visibility private
+          ENV_ANALYTICS_ENABLED = 'DD_TRACE_ETHON_ANALYTICS_ENABLED'
+          ENV_ANALYTICS_SAMPLE_RATE = 'DD_TRACE_ETHON_ANALYTICS_SAMPLE_RATE'
+          DEFAULT_PEER_SERVICE_NAME = 'ethon'
+          SPAN_REQUEST = 'ethon.request'
+          SPAN_MULTI_REQUEST = 'ethon.multi.request'
+          NOT_APPLICABLE_METHOD = 'N/A'
+          TAG_COMPONENT = 'ethon'
+          TAG_OPERATION_REQUEST = 'request'
+          TAG_OPERATION_MULTI_REQUEST = 'multi.request'
+          PEER_SERVICE_SOURCES = Array[
+            Tracing::Metadata::Ext::TAG_PEER_HOSTNAME,
+            Tracing::Metadata::Ext::NET::TAG_DESTINATION_NAME,
+            Tracing::Metadata::Ext::NET::TAG_TARGET_HOST,].freeze
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/contrib/ethon/integration.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require_relative '../integration'
+require_relative 'configuration/settings'
+require_relative '../configuration/resolvers/pattern_resolver'
+require_relative 'patcher'
+
+module Datadog
+  module Tracing
+    module Contrib
+      module Ethon
+        # Description of Ethon integration
+        class Integration
+          include Contrib::Integration
+
+          MINIMUM_VERSION = Gem::Version.new('0.11.0')
+
+          # @public_api Changing the integration name or integration options can cause breaking changes
+          register_as :ethon
+
+          def self.version
+            Gem.loaded_specs['ethon'] && Gem.loaded_specs['ethon'].version
+          end
+
+          def self.loaded?
+            !defined?(::Ethon::Easy).nil?
+          end
+
+          def self.compatible?
+            super && version >= MINIMUM_VERSION
+          end
+
+          def new_configuration
+            Configuration::Settings.new
+          end
+
+          def patcher
+            Patcher
+          end
+
+          def resolver
+            @resolver ||= Contrib::Configuration::Resolvers::PatternResolver.new
+          end
+        end
+      end
+    end
+  end
+end