ddtrace 0.35.2 → 1.1.0

data/lib/datadog/tracing/contrib/kafka/ext.rb

@@ -0,0 +1,52 @@
+# typed: true
+
+module Datadog
+  module Tracing
+    module Contrib
+      module Kafka
+        # Kafka integration constants
+        # @public_api Changing resource names, tag names, or environment variables creates breaking changes.
+        module Ext
+          ENV_ENABLED = 'DD_TRACE_KAFKA_ENABLED'.freeze
+          ENV_ANALYTICS_ENABLED = 'DD_TRACE_KAFKA_ANALYTICS_ENABLED'.freeze
+          ENV_ANALYTICS_SAMPLE_RATE = 'DD_TRACE_KAFKA_ANALYTICS_SAMPLE_RATE'.freeze
+          SPAN_CONNECTION_REQUEST = 'kafka.connection.request'.freeze
+          SPAN_CONSUMER_HEARTBEAT = 'kafka.consumer.heartbeat'.freeze
+          SPAN_CONSUMER_JOIN_GROUP = 'kafka.consumer.join_group'.freeze
+          SPAN_CONSUMER_LEAVE_GROUP = 'kafka.consumer.leave_group'.freeze
+          SPAN_CONSUMER_SYNC_GROUP = 'kafka.consumer.sync_group'.freeze
+          SPAN_DELIVER_MESSAGES = 'kafka.producer.deliver_messages'.freeze
+          SPAN_PROCESS_BATCH = 'kafka.consumer.process_batch'.freeze
+          SPAN_PROCESS_MESSAGE = 'kafka.consumer.process_message'.freeze
+          SPAN_SEND_MESSAGES = 'kafka.producer.send_messages'.freeze
+          TAG_ATTEMPTS = 'kafka.attempts'.freeze
+          TAG_API = 'kafka.api'.freeze
+          TAG_CLIENT = 'kafka.client'.freeze
+          TAG_GROUP = 'kafka.group'.freeze
+          TAG_HIGHWATER_MARK_OFFSET = 'kafka.highwater_mark_offset'.freeze
+          TAG_MESSAGE_COUNT = 'kafka.message_count'.freeze
+          TAG_MESSAGE_KEY = 'kafka.message_key'.freeze
+          TAG_DELIVERED_MESSAGE_COUNT = 'kafka.delivered_message_count'.freeze
+          TAG_OFFSET = 'kafka.offset'.freeze
+          TAG_OFFSET_LAG = 'kafka.offset_lag'.freeze
+          TAG_PARTITION = 'kafka.partition'.freeze
+          TAG_REQUEST_SIZE = 'kafka.request_size'.freeze
+          TAG_RESPONSE_SIZE = 'kafka.response_size'.freeze
+          TAG_SENT_MESSAGE_COUNT = 'kafka.sent_message_count'.freeze
+          TAG_TOPIC = 'kafka.topic'.freeze
+          TAG_TOPIC_PARTITIONS = 'kafka.topic_partitions'.freeze
+          TAG_COMPONENT = 'kafka'.freeze
+          TAG_OPERATION_CONNECTION_REQUEST = 'connection.request'.freeze
+          TAG_OPERATION_CONSUMER_HEARTBEAT = 'consumer.heartbeat'.freeze
+          TAG_OPERATION_CONSUMER_JOIN_GROUP = 'consumer.join_group'.freeze
+          TAG_OPERATION_CONSUMER_LEAVE_GROUP = 'consumer.leave_group'.freeze
+          TAG_OPERATION_CONSUMER_SYNC_GROUP = 'consumer.sync_group'.freeze
+          TAG_OPERATION_DELIVER_MESSAGES = 'producer.deliver_messages'.freeze
+          TAG_OPERATION_PROCESS_BATCH = 'consumer.process_batch'.freeze
+          TAG_OPERATION_PROCESS_MESSAGE = 'consumer.process_message'.freeze
+          TAG_OPERATION_SEND_MESSAGES = 'producer.send_messages'.freeze
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/contrib/kafka/integration.rb

@@ -0,0 +1,44 @@
+# typed: false
+
+require 'datadog/tracing/contrib/integration'
+require 'datadog/tracing/contrib/kafka/configuration/settings'
+require 'datadog/tracing/contrib/kafka/patcher'
+
+module Datadog
+  module Tracing
+    module Contrib
+      module Kafka
+        # Description of Kafka integration
+        class Integration
+          include Contrib::Integration
+
+          MINIMUM_VERSION = Gem::Version.new('0.7.10')
+
+          # @public_api Changing the integration name or integration options can cause breaking changes
+          register_as :kafka, auto_patch: false
+
+          def self.version
+            Gem.loaded_specs['ruby-kafka'] && Gem.loaded_specs['ruby-kafka'].version
+          end
+
+          def self.loaded?
+            !defined?(::Kafka).nil? \
+              && !defined?(::ActiveSupport::Notifications).nil?
+          end
+
+          def self.compatible?
+            super && version >= MINIMUM_VERSION
+          end
+
+          def new_configuration
+            Configuration::Settings.new
+          end
+
+          def patcher
+            Patcher
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/contrib/kafka/patcher.rb

@@ -0,0 +1,29 @@
+# typed: true
+
+require 'datadog/tracing/contrib/patcher'
+require 'datadog/tracing/contrib/kafka/ext'
+require 'datadog/tracing/contrib/kafka/events'
+
+module Datadog
+  module Tracing
+    module Contrib
+      module Kafka
+        # Patcher enables patching of 'kafka' module.
+        module Patcher
+          include Contrib::Patcher
+
+          module_function
+
+          def target_version
+            Integration.version
+          end
+
+          def patch
+            # Subscribe to Kafka events
+            Events.subscribe!
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/contrib/lograge/configuration/settings.rb

@@ -0,0 +1,23 @@
+# typed: false
+
+require 'datadog/tracing/contrib/configuration/settings'
+require 'datadog/tracing/contrib/lograge/ext'
+
+module Datadog
+  module Tracing
+    module Contrib
+      module Lograge
+        module Configuration
+          # Custom settings for the Lograge integration
+          # @public_api
+          class Settings < Contrib::Configuration::Settings
+            option :enabled do |o|
+              o.default { env_to_bool(Ext::ENV_ENABLED, true) }
+              o.lazy
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/contrib/lograge/ext.rb

@@ -0,0 +1,15 @@
+# typed: true
+
+module Datadog
+  module Tracing
+    module Contrib
+      module Lograge
+        # Lograge integration constants
+        # @public_api Changing resource names, tag names, or environment variables creates breaking changes.
+        module Ext
+          ENV_ENABLED = 'DD_TRACE_LOGRAGE_ENABLED'.freeze
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/contrib/lograge/instrumentation.rb

@@ -0,0 +1,47 @@
+# typed: true
+
+require 'datadog/tracing'
+require 'datadog/core/logging/ext'
+
+module Datadog
+  module Tracing
+    module Contrib
+      module Lograge
+        # Instrumentation for Lograge
+        module Instrumentation
+          def self.included(base)
+            base.prepend(InstanceMethods)
+          end
+
+          # Instance methods for configuration
+          module InstanceMethods
+            def custom_options(event)
+              return super unless Datadog.configuration.tracing[:lograge].enabled
+
+              original_custom_options = super(event)
+
+              # Retrieves trace information for current thread
+              correlation = Tracing.correlation
+              # merge original lambda with datadog context
+
+              datadog_trace_log_hash = {
+                # Adds IDs as tags to log output
+                dd: {
+                  # To preserve precision during JSON serialization, use strings for large numbers
+                  trace_id: correlation.trace_id.to_s,
+                  span_id: correlation.span_id.to_s,
+                  env: correlation.env.to_s,
+                  service: correlation.service.to_s,
+                  version: correlation.version.to_s
+                },
+                ddsource: Core::Logging::Ext::DD_SOURCE
+              }
+
+              datadog_trace_log_hash.merge(original_custom_options)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/contrib/lograge/integration.rb

@@ -0,0 +1,50 @@
+# typed: false
+
+require 'datadog/tracing/contrib/integration'
+require 'datadog/tracing/contrib/lograge/configuration/settings'
+require 'datadog/tracing/contrib/lograge/patcher'
+
+module Datadog
+  module Tracing
+    module Contrib
+      module Lograge
+        # Description of Lograge integration
+        class Integration
+          include Contrib::Integration
+
+          MINIMUM_VERSION = Gem::Version.new('0.11.0')
+
+          # @public_api Changing the integration name or integration options can cause breaking changes
+          register_as :lograge
+
+          def self.version
+            Gem.loaded_specs['lograge'] && Gem.loaded_specs['lograge'].version
+          end
+
+          def self.loaded?
+            !defined?(::Lograge::LogSubscribers::Base).nil?
+          end
+
+          def self.compatible?
+            super && version >= MINIMUM_VERSION
+          end
+
+          # enabled by rails integration and has a hard dependancy on rails
+          # so can safely say this shouldn't ever be part of auto instrumentation
+          # https://github.com/roidrage/lograge/blob/1729eab7956bb95c5992e4adab251e4f93ff9280/lograge.gemspec#L18-L20
+          def auto_instrument?
+            false
+          end
+
+          def new_configuration
+            Configuration::Settings.new
+          end
+
+          def patcher
+            Patcher
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/contrib/lograge/patcher.rb

@@ -0,0 +1,29 @@
+# typed: true
+
+require 'datadog/tracing/contrib/patcher'
+require 'datadog/tracing/contrib/lograge/instrumentation'
+
+module Datadog
+  module Tracing
+    module Contrib
+      # Datadog Lograge integration.
+      module Lograge
+        # Patcher enables patching of 'lograge' module.
+        module Patcher
+          include Contrib::Patcher
+
+          module_function
+
+          def target_version
+            Integration.version
+          end
+
+          # patch applies our patch
+          def patch
+            ::Lograge::LogSubscribers::Base.include(Instrumentation)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/contrib/mongodb/configuration/settings.rb

@@ -0,0 +1,38 @@
+# typed: false
+
+require 'datadog/tracing/contrib/configuration/settings'
+require 'datadog/tracing/contrib/mongodb/ext'
+
+module Datadog
+  module Tracing
+    module Contrib
+      module MongoDB
+        module Configuration
+          # Custom settings for the MongoDB integration
+          # @public_api
+          class Settings < Contrib::Configuration::Settings
+            DEFAULT_QUANTIZE = { show: [:collection, :database, :operation] }.freeze
+
+            option :enabled do |o|
+              o.default { env_to_bool(Ext::ENV_ENABLED, true) }
+              o.lazy
+            end
+
+            option :analytics_enabled do |o|
+              o.default { env_to_bool(Ext::ENV_ANALYTICS_ENABLED, false) }
+              o.lazy
+            end
+
+            option :analytics_sample_rate do |o|
+              o.default { env_to_float(Ext::ENV_ANALYTICS_SAMPLE_RATE, 1.0) }
+              o.lazy
+            end
+
+            option :quantize, default: DEFAULT_QUANTIZE
+            option :service_name, default: Ext::DEFAULT_PEER_SERVICE_NAME
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/contrib/mongodb/ext.rb

@@ -0,0 +1,27 @@
+# typed: true
+
+module Datadog
+  module Tracing
+    module Contrib
+      module MongoDB
+        # MongoDB integration constants
+        # @public_api Changing resource names, tag names, or environment variables creates breaking changes.
+        module Ext
+          ENV_ENABLED = 'DD_TRACE_MONGO_ENABLED'.freeze
+          ENV_ANALYTICS_ENABLED = 'DD_TRACE_MONGO_ANALYTICS_ENABLED'.freeze
+          ENV_ANALYTICS_SAMPLE_RATE = 'DD_TRACE_MONGO_ANALYTICS_SAMPLE_RATE'.freeze
+          DEFAULT_PEER_SERVICE_NAME = 'mongodb'.freeze
+          SPAN_COMMAND = 'mongo.cmd'.freeze
+          SPAN_TYPE_COMMAND = 'mongodb'.freeze
+          TAG_COLLECTION = 'mongodb.collection'.freeze
+          TAG_DB = 'mongodb.db'.freeze
+          TAG_OPERATION = 'mongodb.operation'.freeze
+          TAG_QUERY = 'mongodb.query'.freeze
+          TAG_ROWS = 'mongodb.rows'.freeze
+          TAG_COMPONENT = 'mongodb'.freeze
+          TAG_OPERATION_COMMAND = 'command'.freeze
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/contrib/mongodb/instrumentation.rb

@@ -0,0 +1,47 @@
+# typed: false
+
+require 'datadog/tracing/contrib/mongodb/ext'
+require 'datadog/tracing/contrib/mongodb/parsers'
+require 'datadog/tracing/contrib/mongodb/subscribers'
+
+module Datadog
+  module Tracing
+    module Contrib
+      module MongoDB
+        # Instrumentation for Mongo integration
+        module Instrumentation
+          # Instrumentation for Mongo::Client
+          module Client
+            def self.included(base)
+              base.include(InstanceMethods)
+            end
+
+            # Instance methods for Mongo::Client
+            module InstanceMethods
+              def datadog_pin
+                # safe-navigation to avoid crashes during each query
+                return unless respond_to? :cluster
+                return unless cluster.respond_to? :addresses
+                return unless cluster.addresses.respond_to? :first
+
+                Datadog.configuration_for(cluster.addresses.first)
+              end
+
+              def datadog_pin=(pin)
+                # safe-navigation to avoid crashes during each query
+                return unless respond_to? :cluster
+                return unless cluster.respond_to? :addresses
+                return unless cluster.addresses.respond_to? :each
+
+                # attach the PIN to all cluster addresses. One of them is used
+                # when executing a Command and it is attached to the Monitoring
+                # Event instance.
+                cluster.addresses.each { |x| pin.onto(x) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/contrib/mongodb/integration.rb

@@ -0,0 +1,48 @@
+# typed: false
+
+require 'datadog/tracing/contrib/integration'
+require 'datadog/tracing/contrib/configuration/resolvers/pattern_resolver'
+require 'datadog/tracing/contrib/mongodb/configuration/settings'
+require 'datadog/tracing/contrib/mongodb/patcher'
+
+module Datadog
+  module Tracing
+    module Contrib
+      module MongoDB
+        # Description of MongoDB integration
+        class Integration
+          include Contrib::Integration
+
+          MINIMUM_VERSION = Gem::Version.new('2.1.0')
+
+          # @public_api Changing the integration name or integration options can cause breaking changes
+          register_as :mongo, auto_patch: true
+
+          def self.version
+            Gem.loaded_specs['mongo'] && Gem.loaded_specs['mongo'].version
+          end
+
+          def self.loaded?
+            !defined?(::Mongo::Monitoring::Global).nil?
+          end
+
+          def self.compatible?
+            super && version >= MINIMUM_VERSION
+          end
+
+          def new_configuration
+            Configuration::Settings.new
+          end
+
+          def patcher
+            Patcher
+          end
+
+          def resolver
+            @resolver ||= Contrib::Configuration::Resolvers::PatternResolver.new
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/contrib/mongodb/parsers.rb

@@ -0,0 +1,49 @@
+require 'datadog/tracing/contrib/utils/quantization/hash'
+
+# typed: true
+
+module Datadog
+  module Tracing
+    module Contrib
+      # MongoDB module includes classes and functions to instrument MongoDB clients
+      module MongoDB
+        EXCLUDE_KEYS = [:_id].freeze
+        SHOW_KEYS = [].freeze
+        DEFAULT_OPTIONS = { exclude: EXCLUDE_KEYS, show: SHOW_KEYS }.freeze
+
+        module_function
+
+        # skipped keys are related to command names, since they are already
+        # extracted by the query_builder
+        PLACEHOLDER = '?'.freeze
+
+        # returns a formatted and normalized query
+        def query_builder(command_name, database_name, command)
+          # always exclude the command name
+          options = Contrib::Utils::Quantization::Hash.merge_options(quantization_options, exclude: [command_name.to_s])
+
+          # quantized statements keys are strings to avoid leaking Symbols in older Rubies
+          # as Symbols are not GC'ed in Rubies prior to 2.2
+          base_info = Contrib::Utils::Quantization::Hash.format(
+            {
+              'operation' => command_name,
+              'database' => database_name,
+              'collection' => command.values.first
+            },
+            options
+          )
+
+          base_info.merge(Contrib::Utils::Quantization::Hash.format(command, options))
+        end
+
+        def quantization_options
+          Contrib::Utils::Quantization::Hash.merge_options(DEFAULT_OPTIONS, configuration[:quantize])
+        end
+
+        def configuration
+          Datadog.configuration.tracing[:mongo]
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/contrib/mongodb/patcher.rb

@@ -0,0 +1,34 @@
+# typed: false
+
+require 'datadog/tracing/contrib/patcher'
+require 'datadog/tracing/contrib/mongodb/ext'
+require 'datadog/tracing/contrib/mongodb/instrumentation'
+
+module Datadog
+  module Tracing
+    module Contrib
+      module MongoDB
+        # Patcher enables patching of 'mongo' module.
+        module Patcher
+          include Contrib::Patcher
+
+          module_function
+
+          def target_version
+            Integration.version
+          end
+
+          def patch
+            ::Mongo::Client.include(Instrumentation::Client)
+            add_mongo_monitoring
+          end
+
+          def add_mongo_monitoring
+            # Subscribe to all COMMAND queries with our subscriber class
+            ::Mongo::Monitoring::Global.subscribe(::Mongo::Monitoring::COMMAND, MongoCommandSubscriber.new)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/contrib/mongodb/subscribers.rb

@@ -0,0 +1,121 @@
+# typed: true
+
+require 'datadog/tracing/contrib/analytics'
+require 'datadog/tracing/contrib/mongodb/ext'
+require 'datadog/tracing/contrib/mongodb/parsers'
+require 'datadog/tracing/metadata/ext'
+
+module Datadog
+  module Tracing
+    module Contrib
+      module MongoDB
+        # `MongoCommandSubscriber` listens to all events from the `Monitoring`
+        # system available in the Mongo driver.
+        class MongoCommandSubscriber
+          def started(event)
+            return unless Tracing.enabled?
+
+            service = Datadog.configuration_for(event.address, :service_name) \
+                      || Datadog.configuration.tracing[:mongo, event.address.seed][:service_name]
+
+            # start a trace and store it in the current thread; using the `operation_id`
+            # is safe since it's a unique id used to link events together. Also only one
+            # thread is involved in this execution so thread-local storage should be safe. Reference:
+            # https://github.com/mongodb/mongo-ruby-driver/blob/master/lib/mongo/monitoring.rb#L70
+            # https://github.com/mongodb/mongo-ruby-driver/blob/master/lib/mongo/monitoring/publishable.rb#L38-L56
+            span = Tracing.trace(Ext::SPAN_COMMAND, service: service, span_type: Ext::SPAN_TYPE_COMMAND)
+            set_span(event, span)
+
+            # build a quantized Query using the Parser module
+            query = MongoDB.query_builder(event.command_name, event.database_name, event.command)
+            serialized_query = query.to_s
+
+            span.set_tag(Tracing::Metadata::Ext::TAG_COMPONENT, Ext::TAG_COMPONENT)
+            span.set_tag(Tracing::Metadata::Ext::TAG_OPERATION, Ext::TAG_OPERATION_COMMAND)
+
+            # Tag as an external peer service
+            span.set_tag(Tracing::Metadata::Ext::TAG_PEER_SERVICE, span.service)
+            span.set_tag(Tracing::Metadata::Ext::TAG_PEER_HOSTNAME, event.address.host)
+
+            # Set analytics sample rate
+            Contrib::Analytics.set_sample_rate(span, analytics_sample_rate) if analytics_enabled?
+
+            # add operation tags; the full query is stored and used as a resource,
+            # since it has been quantized and reduced
+            span.set_tag(Ext::TAG_DB, query['database'])
+            span.set_tag(Ext::TAG_COLLECTION, query['collection'])
+            span.set_tag(Ext::TAG_OPERATION, query['operation'])
+            span.set_tag(Ext::TAG_QUERY, serialized_query)
+            span.set_tag(Tracing::Metadata::Ext::NET::TAG_TARGET_HOST, event.address.host)
+            span.set_tag(Tracing::Metadata::Ext::NET::TAG_TARGET_PORT, event.address.port)
+
+            # set the resource with the quantized query
+            span.resource = serialized_query
+          end
+
+          def failed(event)
+            span = get_span(event)
+            return unless span
+
+            # the failure is not a real exception because it's handled by
+            # the framework itself, so we set only the error and the message
+            span.set_error(event)
+          rescue StandardError => e
+            Datadog.logger.debug("error when handling MongoDB 'failed' event: #{e}")
+          ensure
+            # whatever happens, the Span must be removed from the local storage and
+            # it must be finished to prevent any leak
+            span.finish unless span.nil?
+            clear_span(event)
+          end
+
+          def succeeded(event)
+            span = get_span(event)
+            return unless span
+
+            # add fields that are available only after executing the query
+            rows = event.reply.fetch('n', nil)
+            span.set_tag(Ext::TAG_ROWS, rows) unless rows.nil?
+          rescue StandardError => e
+            Datadog.logger.debug("error when handling MongoDB 'succeeded' event: #{e}")
+          ensure
+            # whatever happens, the Span must be removed from the local storage and
+            # it must be finished to prevent any leak
+            span.finish unless span.nil?
+            clear_span(event)
+          end
+
+          private
+
+          def get_span(event)
+            Thread.current[:datadog_mongo_span] \
+              && Thread.current[:datadog_mongo_span][event.request_id]
+          end
+
+          def set_span(event, span)
+            Thread.current[:datadog_mongo_span] ||= {}
+            Thread.current[:datadog_mongo_span][event.request_id] = span
+          end
+
+          def clear_span(event)
+            return if Thread.current[:datadog_mongo_span].nil?
+
+            Thread.current[:datadog_mongo_span].delete(event.request_id)
+          end
+
+          def analytics_enabled?
+            Contrib::Analytics.enabled?(datadog_configuration[:analytics_enabled])
+          end
+
+          def analytics_sample_rate
+            datadog_configuration[:analytics_sample_rate]
+          end
+
+          def datadog_configuration
+            Datadog.configuration.tracing[:mongo]
+          end
+        end
+      end
+    end
+  end
+end