datadog 2.0.0.beta1

data/lib/datadog/tracing/contrib.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require_relative '../tracing'
+require_relative 'contrib/registry'
+require_relative 'contrib/extensions'
+
+module Datadog
+  module Tracing
+    module Contrib
+      # Registry is a global, declarative repository of all available integrations.
+      #
+      # Integrations should register themselves with the registry as early as
+      # possible as the initial tracer configuration can only activate integrations
+      # if they have already been registered.
+      #
+      # Despite that, integrations *can* be appended to the registry at any point
+      # of the program execution. Newly appended integrations can then be
+      # activated during tracer reconfiguration.
+      #
+      # The registry does not depend on runtime configuration and registered integrations
+      # must execute correctly after successive configuration changes.
+      # The registered integrations themselves can depend on the stateful configuration
+      # of the tracer.
+      #
+      # `Datadog.registry` is a helper accessor to this constant, but it's only available
+      # after the tracer has complete initialization. Use `Datadog::Tracing::Contrib::REGISTRY` instead
+      # of `Datadog.registry` when you code might be called during tracer initialization.
+      REGISTRY = Registry.new
+    end
+  end
+end
+
+require_relative 'contrib/action_cable/integration'
+require_relative 'contrib/action_mailer/integration'
+require_relative 'contrib/action_pack/integration'
+require_relative 'contrib/action_view/integration'
+require_relative 'contrib/active_model_serializers/integration'
+require_relative 'contrib/active_job/integration'
+require_relative 'contrib/active_record/integration'
+require_relative 'contrib/active_support/integration'
+require_relative 'contrib/aws/integration'
+require_relative 'contrib/concurrent_ruby/integration'
+require_relative 'contrib/dalli/integration'
+require_relative 'contrib/delayed_job/integration'
+require_relative 'contrib/elasticsearch/integration'
+require_relative 'contrib/ethon/integration'
+require_relative 'contrib/excon/integration'
+require_relative 'contrib/faraday/integration'
+require_relative 'contrib/grape/integration'
+require_relative 'contrib/graphql/integration'
+require_relative 'contrib/grpc'
+require_relative 'contrib/hanami/integration'
+require_relative 'contrib/http'
+require_relative 'contrib/httpclient/integration'
+require_relative 'contrib/httprb/integration'
+require_relative 'contrib/integration'
+require_relative 'contrib/kafka/integration'
+require_relative 'contrib/lograge/integration'
+require_relative 'contrib/mongodb/integration'
+require_relative 'contrib/mysql2/integration'
+require_relative 'contrib/opensearch/integration'
+require_relative 'contrib/pg/integration'
+require_relative 'contrib/presto/integration'
+require_relative 'contrib/que/integration'
+require_relative 'contrib/racecar/integration'
+require_relative 'contrib/rack/integration'
+require_relative 'contrib/rails/integration'
+require_relative 'contrib/rake/integration'
+require_relative 'contrib/redis/integration'
+require_relative 'contrib/resque/integration'
+require_relative 'contrib/rest_client/integration'
+require_relative 'contrib/roda/integration'
+require_relative 'contrib/semantic_logger/integration'
+require_relative 'contrib/sequel/integration'
+require_relative 'contrib/shoryuken/integration'
+require_relative 'contrib/sidekiq'
+require_relative 'contrib/sinatra/integration'
+require_relative 'contrib/sneakers/integration'
+require_relative 'contrib/stripe/integration'
+require_relative 'contrib/sucker_punch/integration'
+require_relative 'contrib/trilogy/integration'

data/lib/datadog/tracing/correlation.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require_relative 'utils'
+require_relative 'metadata/ext'
+require_relative '../core/logging/ext'
+
+module Datadog
+  module Tracing
+    # Contains behavior for managing correlations with tracing
+    # e.g. Retrieve a correlation to the current trace for logging, etc.
+    # This class is for usage with log correlation.
+    # To continue from a trace, users should use TraceDigest instead.
+    module Correlation
+      # Represents current trace state with key identifiers
+      # @public_api
+      class Identifier
+        LOG_ATTR_ENV = 'dd.env'
+        LOG_ATTR_SERVICE = 'dd.service'
+        LOG_ATTR_SPAN_ID = 'dd.span_id'
+        LOG_ATTR_TRACE_ID = 'dd.trace_id'
+        LOG_ATTR_VERSION = 'dd.version'
+        LOG_ATTR_SOURCE = 'ddsource'
+
+        attr_reader \
+          :env,
+          :service,
+          :span_id,
+          :version
+
+        # @!visibility private
+        def initialize(
+          env: nil,
+          service: nil,
+          span_id: nil,
+          trace_id: nil,
+          version: nil
+        )
+          # Dup and freeze strings so they aren't modified by reference.
+          @env = Core::Utils::SafeDup.frozen_dup(env || Datadog.configuration.env)
+          @service = Core::Utils::SafeDup.frozen_dup(service || Datadog.configuration.service)
+          @span_id = (span_id || 0).to_s
+          @trace_id = trace_id || 0
+          @version = Core::Utils::SafeDup.frozen_dup(version || Datadog.configuration.version)
+        end
+
+        def to_h
+          @to_h ||= {
+            # Adds IDs as tags to log output
+            dd: {
+              # To preserve precision during JSON serialization, use strings for large numbers
+              env: env.to_s,
+              service: service.to_s,
+              version: version.to_s,
+              trace_id: trace_id.to_s,
+              span_id: span_id.to_s
+            },
+            ddsource: Core::Logging::Ext::DD_SOURCE
+          }
+        end
+
+        # This method (#to_log_format) implements an algorithm by prefixing keys for nested values
+        # but the algorithm makes the constants implicit. Hence, we use it for validation during test.
+        def to_log_format
+          @log_format ||= begin
+            attributes = []
+            attributes << "#{LOG_ATTR_ENV}=#{env}" unless env.nil?
+            attributes << "#{LOG_ATTR_SERVICE}=#{service}"
+            attributes << "#{LOG_ATTR_VERSION}=#{version}" unless version.nil?
+            attributes << "#{LOG_ATTR_TRACE_ID}=#{trace_id}"
+            attributes << "#{LOG_ATTR_SPAN_ID}=#{span_id}"
+            attributes << "#{LOG_ATTR_SOURCE}=#{Core::Logging::Ext::DD_SOURCE}"
+            attributes.join(' ')
+          end
+        end
+
+        def trace_id
+          if Datadog.configuration.tracing.trace_id_128_bit_logging_enabled &&
+              !Tracing::Utils::TraceId.to_high_order(@trace_id).zero?
+            Kernel.format('%032x', @trace_id)
+          else
+            Tracing::Utils::TraceId.to_low_order(@trace_id).to_s
+          end
+        end
+      end
+
+      module_function
+
+      # Produces a CorrelationIdentifier from the TraceDigest provided
+      #
+      # DEV: can we memoize this object, give it can be common to
+      # use a correlation multiple times, specially in the context of logging?
+      # @!visibility private
+      def identifier_from_digest(digest)
+        return Identifier.new unless digest
+
+        Identifier.new(
+          span_id: digest.span_id,
+          trace_id: digest.trace_id,
+        )
+      end
+    end
+  end
+end

data/lib/datadog/tracing/diagnostics/environment_logger.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+require 'date'
+require 'json'
+require 'rbconfig'
+require_relative '../../core/diagnostics/environment_logger'
+
+module Datadog
+  module Tracing
+    module Diagnostics
+      # Collects and logs Tracing diagnostic and error information
+      module EnvironmentLogger
+        extend Core::Diagnostics::EnvironmentLogging
+
+        def self.collect_and_log!(responses: nil)
+          if log?
+            env_data = EnvironmentCollector.collect_config!
+            log_configuration!('TRACING', env_data.to_json)
+
+            if responses
+              err_data = EnvironmentCollector.collect_errors!(responses)
+              err_data.reject! { |_, v| v.nil? } # Remove empty values from hash output
+              log_error!('TRACING', 'Agent Error', err_data.to_json) unless err_data.empty?
+            end
+          end
+        rescue => e
+          logger.warn("Failed to collect tracing environment information: #{e} Location: #{Array(e.backtrace).first}")
+        end
+      end
+
+      # Collects environment information for Tracing diagnostic logging
+      module EnvironmentCollector
+        class << self
+          def collect_config!
+            {
+              enabled: enabled,
+              agent_url: agent_url,
+              analytics_enabled: analytics_enabled,
+              sample_rate: sample_rate,
+              sampling_rules: sampling_rules,
+              integrations_loaded: integrations_loaded,
+              partial_flushing_enabled: partial_flushing_enabled,
+              **instrumented_integrations_settings
+            }
+          end
+
+          def collect_errors!(responses)
+            {
+              agent_error: agent_error(responses)
+            }
+          end
+
+          # @return [Boolean, nil]
+          def enabled
+            !!Datadog.configuration.tracing.enabled
+          end
+
+          # @return [String, nil] target agent URL for trace flushing
+          def agent_url
+            # Retrieve the effect agent URL, regardless of how it was configured
+            transport = Tracing.send(:tracer).writer.transport
+
+            # return `nil` with IO transport
+            return unless transport.respond_to?(:client)
+
+            adapter = transport.client.api.adapter
+            adapter.url
+          end
+
+          # Error returned by Datadog agent during a tracer flush attempt
+          # @return [String] concatenated list of transport errors
+          def agent_error(responses)
+            error_responses = responses.reject(&:ok?)
+
+            return nil if error_responses.empty?
+
+            error_responses.map(&:inspect).join(',')
+          end
+
+          # @return [Boolean, nil] analytics enabled in configuration
+          def analytics_enabled
+            !!Datadog.configuration.tracing.analytics.enabled
+          end
+
+          # @return [Numeric, nil] tracer sample rate configured
+          def sample_rate
+            sampler = Datadog.configuration.tracing.sampler
+            return nil unless sampler
+
+            sampler.sample_rate(nil) rescue nil
+          end
+
+          # DEV: We currently only support SimpleRule instances.
+          # DEV: These are the most commonly used rules.
+          # DEV: We should expand support for other rules in the future,
+          # DEV: although it is tricky to serialize arbitrary rules.
+          #
+          # @return [Hash, nil] sample rules configured
+          def sampling_rules
+            sampler = Datadog.configuration.tracing.sampler
+            return nil unless sampler.is_a?(Tracing::Sampling::PrioritySampler) &&
+              sampler.priority_sampler.is_a?(Tracing::Sampling::RuleSampler)
+
+            sampler.priority_sampler.rules.map do |rule|
+              next unless rule.is_a?(Tracing::Sampling::SimpleRule)
+
+              {
+                name: rule.matcher.name,
+                service: rule.matcher.service,
+                sample_rate: rule.sampler.sample_rate(nil)
+              }
+            end.compact
+          end
+
+          # Concatenated list of integrations activated, with their gem version.
+          # Example: "rails@6.0.3,rack@2.2.3"
+          #
+          # @return [String, nil]
+          def integrations_loaded
+            integrations = instrumented_integrations
+            return if integrations.empty?
+
+            integrations.map { |name, integration| "#{name}@#{integration.class.version}" }.join(',')
+          end
+
+          # @return [Boolean, nil] partial flushing enabled in configuration
+          def partial_flushing_enabled
+            !!Datadog.configuration.tracing.partial_flush.enabled
+          end
+
+          private
+
+          def instrumented_integrations
+            # `instrumented_integrations` method is added only if tracing contrib extensions are required.
+            # If tracing is used in manual mode without integrations enabled this method does not exist.
+            # CI visibility gem datadog-ci uses Datadog::Tracer without requiring datadog/tracing/contrib folder
+            # nor enabling any integrations by default which causes EnvironmentCollector to fail.
+            return {} unless Datadog.configuration.tracing.respond_to?(:instrumented_integrations)
+
+            Datadog.configuration.tracing.instrumented_integrations
+          end
+
+          # Capture all active integration settings into "integrationName_settingName: value" entries.
+          def instrumented_integrations_settings
+            instrumented_integrations.flat_map do |name, integration|
+              integration.configuration.to_h.flat_map do |setting, value|
+                next [] if setting == :tracer # Skip internal Ruby objects
+
+                # Convert value to a string to avoid custom #to_json
+                # handlers possibly causing errors.
+                [[:"integration_#{name}_#{setting}", value.to_s]]
+              end
+            end.to_h
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/diagnostics/ext.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Tracing
+    module Diagnostics
+      # @public_api
+      module Ext
+        # Health
+        module Health
+          # Metrics
+          module Metrics
+            METRIC_API_ERRORS = 'datadog.tracer.api.errors'
+            METRIC_API_REQUESTS = 'datadog.tracer.api.requests'
+            METRIC_API_RESPONSES = 'datadog.tracer.api.responses'
+            METRIC_ERROR_CONTEXT_OVERFLOW = 'datadog.tracer.error.context_overflow'
+            METRIC_ERROR_INSTRUMENTATION_PATCH = 'datadog.tracer.error.instrumentation_patch'
+            METRIC_ERROR_SPAN_FINISH = 'datadog.tracer.error.span_finish'
+            METRIC_ERROR_UNFINISHED_SPANS = 'datadog.tracer.error.unfinished_spans'
+            METRIC_INSTRUMENTATION_PATCHED = 'datadog.tracer.instrumentation_patched'
+            METRIC_QUEUE_ACCEPTED = 'datadog.tracer.queue.accepted'
+            METRIC_QUEUE_ACCEPTED_LENGTHS = 'datadog.tracer.queue.accepted_lengths'
+            METRIC_QUEUE_DROPPED = 'datadog.tracer.queue.dropped'
+            METRIC_QUEUE_LENGTH = 'datadog.tracer.queue.length'
+            METRIC_QUEUE_MAX_LENGTH = 'datadog.tracer.queue.max_length'
+            METRIC_QUEUE_SPANS = 'datadog.tracer.queue.spans'
+            METRIC_SAMPLING_SERVICE_CACHE_LENGTH = 'datadog.tracer.sampling.service_cache_length'
+            METRIC_TRACES_FILTERED = 'datadog.tracer.traces.filtered'
+            METRIC_TRANSPORT_CHUNKED = 'datadog.tracer.transport.chunked'
+            METRIC_TRANSPORT_TRACE_TOO_LARGE = 'datadog.tracer.transport.trace_too_large'
+            METRIC_WRITER_CPU_TIME = 'datadog.tracer.writer.cpu_time'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/diagnostics/health.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require_relative 'ext'
+
+module Datadog
+  module Tracing
+    module Diagnostics
+      # Health-related diagnostics
+      module Health
+        # Health metrics for diagnostics
+        module Metrics
+          def self.extended(base)
+            base.class_eval do
+              count :api_errors, Ext::Health::Metrics::METRIC_API_ERRORS
+              count :api_requests, Ext::Health::Metrics::METRIC_API_REQUESTS
+              count :api_responses, Ext::Health::Metrics::METRIC_API_RESPONSES
+              count :error_context_overflow, Ext::Health::Metrics::METRIC_ERROR_CONTEXT_OVERFLOW
+              count :error_instrumentation_patch, Ext::Health::Metrics::METRIC_ERROR_INSTRUMENTATION_PATCH
+              count :error_span_finish, Ext::Health::Metrics::METRIC_ERROR_SPAN_FINISH
+              count :error_unfinished_spans, Ext::Health::Metrics::METRIC_ERROR_UNFINISHED_SPANS
+              count :instrumentation_patched, Ext::Health::Metrics::METRIC_INSTRUMENTATION_PATCHED
+              count :queue_accepted, Ext::Health::Metrics::METRIC_QUEUE_ACCEPTED
+              count :queue_accepted_lengths, Ext::Health::Metrics::METRIC_QUEUE_ACCEPTED_LENGTHS
+              count :queue_dropped, Ext::Health::Metrics::METRIC_QUEUE_DROPPED
+              count :traces_filtered, Ext::Health::Metrics::METRIC_TRACES_FILTERED
+              count :transport_trace_too_large, Ext::Health::Metrics::METRIC_TRANSPORT_TRACE_TOO_LARGE
+              count :transport_chunked, Ext::Health::Metrics::METRIC_TRANSPORT_CHUNKED
+              count :writer_cpu_time, Ext::Health::Metrics::METRIC_WRITER_CPU_TIME
+
+              gauge :queue_length, Ext::Health::Metrics::METRIC_QUEUE_LENGTH
+              gauge :queue_max_length, Ext::Health::Metrics::METRIC_QUEUE_MAX_LENGTH
+              gauge :queue_spans, Ext::Health::Metrics::METRIC_QUEUE_SPANS
+              gauge :sampling_service_cache_length, Ext::Health::Metrics::METRIC_SAMPLING_SERVICE_CACHE_LENGTH
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/distributed/b3_multi.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require_relative 'helpers'
+require_relative '../trace_digest'
+require_relative '../utils'
+
+module Datadog
+  module Tracing
+    module Distributed
+      # B3 multi header-style trace propagation.
+      # @see https://github.com/openzipkin/b3-propagation#multiple-headers
+      class B3Multi
+        B3_TRACE_ID_KEY = 'x-b3-traceid'
+        B3_SPAN_ID_KEY = 'x-b3-spanid'
+        B3_SAMPLED_KEY = 'x-b3-sampled'
+
+        def initialize(
+          fetcher:,
+          trace_id_key: B3_TRACE_ID_KEY,
+          span_id_key: B3_SPAN_ID_KEY,
+          sampled_key: B3_SAMPLED_KEY
+        )
+          @trace_id_key = trace_id_key
+          @span_id_key = span_id_key
+          @sampled_key = sampled_key
+          @fetcher = fetcher
+        end
+
+        def inject!(digest, data = {})
+          return if digest.nil?
+
+          # DEV: We need these to be hex encoded
+          data[@trace_id_key] = format('%032x', digest.trace_id)
+          data[@span_id_key] = format('%016x', digest.span_id)
+
+          if digest.trace_sampling_priority
+            sampling_priority = Helpers.clamp_sampling_priority(
+              digest.trace_sampling_priority
+            )
+            data[@sampled_key] = sampling_priority.to_s
+          end
+
+          data
+        end
+
+        def extract(data)
+          # DEV: B3 doesn't have "origin"
+          fetcher = @fetcher.new(data)
+
+          trace_id = Helpers.parse_hex_id(fetcher[@trace_id_key])
+
+          # Return early if this propagation is not valid
+          return if trace_id.nil? || trace_id <= 0 || trace_id > Tracing::Utils::TraceId::MAX
+
+          span_id = Helpers.parse_hex_id(fetcher[@span_id_key])
+
+          # Return early if this propagation is not valid
+          return if span_id.nil? || span_id <= 0 || span_id >= Tracing::Utils::EXTERNAL_MAX_ID
+
+          # We don't need to try and convert sampled since B3 supports 0/1 (AUTO_REJECT/AUTO_KEEP)
+          sampling_priority = Helpers.parse_decimal_id(fetcher[@sampled_key])
+
+          TraceDigest.new(
+            trace_id: trace_id,
+            span_id: span_id,
+            trace_sampling_priority: sampling_priority,
+            span_remote: true,
+          )
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/distributed/b3_single.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require_relative 'helpers'
+require_relative '../trace_digest'
+
+module Datadog
+  module Tracing
+    module Distributed
+      # B3 single header-style trace propagation.
+      #
+      # DEV: Format:
+      # DEV:   b3: {TraceId}-{SpanId}-{SamplingState}-{ParentSpanId}
+      # DEV: https://github.com/apache/incubator-zipkin-b3-propagation/tree/7c6e9f14d6627832bd80baa87ac7dabee7be23cf#single-header
+      # DEV: `{SamplingState}` and `{ParentSpanId}` are optional
+      #
+      # @see https://github.com/openzipkin/b3-propagation#single-header
+      class B3Single
+        B3_SINGLE_HEADER_KEY = 'b3'
+
+        def initialize(fetcher:, key: B3_SINGLE_HEADER_KEY)
+          @key = key
+          @fetcher = fetcher
+        end
+
+        def inject!(digest, env)
+          return if digest.nil?
+
+          # DEV: We need these to be hex encoded
+          value = "#{format('%032x', digest.trace_id)}-#{format('%016x', digest.span_id)}"
+
+          if digest.trace_sampling_priority
+            sampling_priority = Helpers.clamp_sampling_priority(
+              digest.trace_sampling_priority
+            )
+            value += "-#{sampling_priority}"
+          end
+
+          env[@key] = value
+          env
+        end
+
+        def extract(env)
+          fetcher = @fetcher.new(env)
+          value = fetcher[@key]
+
+          return unless value
+
+          parts = value.split('-')
+          trace_id = Helpers.parse_hex_id(parts[0]) unless parts.empty?
+          # Return early if this propagation is not valid
+          return if trace_id.nil? || trace_id <= 0 || trace_id > Tracing::Utils::TraceId::MAX
+
+          span_id = Helpers.parse_hex_id(parts[1]) if parts.length > 1
+          # Return early if this propagation is not valid
+          return if span_id.nil? || span_id <= 0 || span_id >= Tracing::Utils::EXTERNAL_MAX_ID
+
+          sampling_priority = Helpers.parse_decimal_id(parts[2]) if parts.length > 2
+
+          TraceDigest.new(
+            span_id: span_id,
+            trace_id: trace_id,
+            trace_sampling_priority: sampling_priority,
+            span_remote: true,
+          )
+        end
+      end
+    end
+  end
+end