ddtrace 0.40.0 → 0.50.0

data/lib/ddtrace/configuration/components.rb

@@ -1,13 +1,17 @@
+require 'ddtrace/configuration/agent_settings_resolver'
 require 'ddtrace/diagnostics/health'
 require 'ddtrace/logger'
+require 'ddtrace/profiling'
 require 'ddtrace/runtime/metrics'
 require 'ddtrace/tracer'
+require 'ddtrace/sync_writer'
 require 'ddtrace/workers/runtime_metrics'
 
 module Datadog
   module Configuration
     # Global components for the trace library.
-    # rubocop:disable Metrics/LineLength
+    # rubocop:disable Metrics/ClassLength
+    # rubocop:disable Layout/LineLength
     class Components
       class << self
         def build_health_metrics(settings)
@@ -19,7 +23,7 @@ module Datadog
         end
 
         def build_logger(settings)
-          logger = settings.logger.instance || Datadog::Logger.new(STDOUT)
+          logger = settings.logger.instance || Datadog::Logger.new($stdout)
           logger.level = settings.diagnostics.debug ? ::Logger::DEBUG : settings.logger.level
 
           logger
@@ -43,7 +47,7 @@ module Datadog
           Datadog::Workers::RuntimeMetrics.new(options)
         end
 
-        def build_tracer(settings)
+        def build_tracer(settings, agent_settings)
           # If a custom tracer has been provided, use it instead.
           # Ignore all other options (they should already be configured.)
           tracer = settings.tracer.instance
@@ -61,11 +65,27 @@ module Datadog
           #       tracer initialization for now. Just reconfigure using the
           #       existing mutable #configure function. Remove when these components
           #       are extracted.
-          tracer.configure(build_tracer_options(settings))
+          tracer.configure(agent_settings: agent_settings, **build_tracer_options(settings, agent_settings))
 
           tracer
         end
 
+        def build_profiler(settings, agent_settings)
+          return unless Datadog::Profiling.supported? && settings.profiling.enabled
+
+          # Load extensions needed to support some of the Profiling features
+          Datadog::Profiling::Tasks::Setup.new.run
+
+          # NOTE: Please update the Initialization section of ProfilingDevelopment.md with any changes to this method
+
+          recorder = build_profiler_recorder(settings)
+          collectors = build_profiler_collectors(settings, recorder)
+          exporters = build_profiler_exporters(settings, agent_settings)
+          scheduler = build_profiler_scheduler(settings, recorder, exporters)
+
+          Datadog::Profiler.new(collectors, scheduler)
+        end
+
         private
 
         def build_tracer_tags(settings)
@@ -75,26 +95,78 @@ module Datadog
           end
         end
 
-        def build_tracer_options(settings)
-          settings = settings.tracer
-
-          {}.tap do |opts|
-            opts[:hostname] = settings.hostname unless settings.hostname.nil?
-            opts[:min_spans_before_partial_flush] = settings.partial_flush.min_spans_threshold unless settings.partial_flush.min_spans_threshold.nil?
-            opts[:partial_flush] = settings.partial_flush.enabled unless settings.partial_flush.enabled.nil?
-            opts[:port] = settings.port unless settings.port.nil?
-            opts[:priority_sampling] = settings.priority_sampling unless settings.priority_sampling.nil?
-            opts[:sampler] = settings.sampler unless settings.sampler.nil?
-            opts[:transport_options] = settings.transport_options
-            opts[:writer] = settings.writer unless settings.writer.nil?
-            opts[:writer_options] = settings.writer_options if settings.writer.nil?
+        def build_tracer_options(settings, agent_settings)
+          tracer_options = {}.tap do |opts|
+            tset = settings.tracer
+            opts[:min_spans_before_partial_flush] = tset.partial_flush.min_spans_threshold unless tset.partial_flush.min_spans_threshold.nil?
+            opts[:partial_flush] = tset.partial_flush.enabled unless tset.partial_flush.enabled.nil?
+            opts[:priority_sampling] = tset.priority_sampling unless tset.priority_sampling.nil?
+            opts[:sampler] = tset.sampler unless tset.sampler.nil?
+            opts[:writer] = tset.writer unless tset.writer.nil?
+            opts[:writer_options] = tset.writer_options if tset.writer.nil?
+          end
+
+          # Apply test mode settings if test mode is activated
+          if settings.test_mode.enabled
+            build_tracer_test_mode_options(tracer_options, settings, agent_settings)
+          else
+            tracer_options
           end
         end
+
+        def build_tracer_test_mode_options(tracer_options, settings, agent_settings)
+          tracer_options.tap do |opts|
+            # Do not sample any spans for tests; all must be preserved.
+            opts[:sampler] = Datadog::AllSampler.new
+
+            # If context flush behavior is provided, use it instead.
+            opts[:context_flush] = settings.test_mode.context_flush if settings.test_mode.context_flush
+
+            # Flush traces synchronously, to guarantee they are written.
+            writer_options = settings.test_mode.writer_options || {}
+            writer_options[:agent_settings] = agent_settings if agent_settings
+            opts[:writer] = Datadog::SyncWriter.new(writer_options)
+          end
+        end
+
+        def build_profiler_recorder(settings)
+          event_classes = [Datadog::Profiling::Events::StackSample]
+
+          Datadog::Profiling::Recorder.new(event_classes, settings.profiling.max_events)
+        end
+
+        def build_profiler_collectors(settings, recorder)
+          [
+            Datadog::Profiling::Collectors::Stack.new(
+              recorder,
+              max_frames: settings.profiling.max_frames
+              # TODO: Provide proc that identifies Datadog worker threads?
+              # ignore_thread: settings.profiling.ignore_profiler
+            )
+          ]
+        end
+
+        def build_profiler_exporters(settings, agent_settings)
+          transport =
+            settings.profiling.exporter.transport || Datadog::Profiling::Transport::HTTP.default(
+              agent_settings: agent_settings,
+              site: settings.site,
+              api_key: settings.api_key,
+              profiling_upload_timeout_seconds: settings.profiling.upload.timeout_seconds
+            )
+
+          [Datadog::Profiling::Exporter.new(transport)]
+        end
+
+        def build_profiler_scheduler(settings, recorder, exporters)
+          Datadog::Profiling::Scheduler.new(recorder, exporters)
+        end
       end
 
       attr_reader \
         :health_metrics,
         :logger,
+        :profiler,
         :runtime_metrics,
         :tracer
 
@@ -102,8 +174,13 @@ module Datadog
         # Logger
         @logger = self.class.build_logger(settings)
 
+        agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
+
         # Tracer
-        @tracer = self.class.build_tracer(settings)
+        @tracer = self.class.build_tracer(settings, agent_settings)
+
+        # Profiler
+        @profiler = self.class.build_profiler(settings, agent_settings)
 
         # Runtime metrics
         @runtime_metrics = self.class.build_runtime_metrics_worker(settings)
@@ -113,7 +190,20 @@ module Datadog
       end
 
       # Starts up components
-      def startup!(settings); end
+      def startup!(settings)
+        if settings.profiling.enabled
+          if profiler
+            @logger.debug('Profiling started')
+            profiler.start
+          else
+            # Display a warning for users who expected profiling to be enabled
+            unsupported_reason = Datadog::Profiling.unsupported_reason
+            logger.warn("Profiling was requested but is not supported, profiling disabled: #{unsupported_reason}")
+          end
+        else
+          @logger.debug('Profiling is disabled')
+        end
+      end
 
       # Shuts down all the components in use.
       # If it has another instance to compare to, it will compare
@@ -123,12 +213,20 @@ module Datadog
         # (e.g. a custom tracer instance passed in.)
         tracer.shutdown! unless replacement && tracer == replacement.tracer
 
+        # Shutdown old profiler
+        profiler.shutdown! unless profiler.nil?
+
         # Shutdown workers
-        runtime_metrics.enabled = false
-        runtime_metrics.stop(true)
+        runtime_metrics.stop(true, close_metrics: false)
 
         # Shutdown the old metrics, unless they are still being used.
         # (e.g. custom Statsd instances.)
+        #
+        # TODO: This violates the encapsulation created by Runtime::Metrics and
+        # Health::Metrics, by directly manipulating `statsd` and changing
+        # it's lifecycle management.
+        # If we need to directly have ownership of `statsd` lifecycle, we should
+        # have direct ownership of it.
         old_statsd = [
           runtime_metrics.metrics.statsd,
           health_metrics.statsd

data/lib/ddtrace/configuration/option_definition.rb

@@ -60,7 +60,7 @@ module Datadog
         end
 
         def default(value = nil, &block)
-          @default = block_given? ? block : value
+          @default = block || value
         end
 
         def delegate_to(&block)
@@ -71,8 +71,6 @@ module Datadog
           @helpers[name] = block
         end
 
-        # rubocop:disable Style/TrivialAccessors
-        # (Rubocop erroneously thinks #lazy == #lazy= )
         def lazy(value = true)
           @lazy = value
         end

data/lib/ddtrace/configuration/options.rb

@@ -7,17 +7,15 @@ module Datadog
     # Behavior for a configuration object that has options
     module Options
       def self.included(base)
-        base.send(:extend, ClassMethods)
-        base.send(:include, InstanceMethods)
+        base.extend(ClassMethods)
+        base.include(InstanceMethods)
       end
 
       # Class behavior for a configuration object with options
       module ClassMethods
         def options
-          @options ||= begin
-            # Allows for class inheritance of option definitions
-            superclass <= Options ? superclass.options.dup : OptionDefinitionSet.new
-          end
+          # Allows for class inheritance of option definitions
+          @options ||= superclass <= Options ? superclass.options.dup : OptionDefinitionSet.new
         end
 
         protected
@@ -26,7 +24,9 @@ module Datadog
           builder = OptionDefinition::Builder.new(name, meta, &block)
           options[name] = builder.to_definition.tap do
             # Resolve and define helper functions
-            helpers = default_helpers(name).merge(builder.helpers)
+            helpers = default_helpers(name)
+            # Prevent unnecessary creation of an identical copy of helpers if there's nothing to merge
+            helpers = helpers.merge(builder.helpers) unless builder.helpers.empty?
             define_helpers(helpers)
           end
         end
@@ -49,6 +49,7 @@ module Datadog
         def define_helpers(helpers)
           helpers.each do |name, block|
             next unless block.is_a?(Proc)
+
             define_method(name, &block)
           end
         end
@@ -100,9 +101,7 @@ module Datadog
         end
 
         def assert_valid_option!(name)
-          unless option_defined?(name)
-            raise(InvalidOptionError, "#{self.class.name} doesn't define the option: #{name}")
-          end
+          raise(InvalidOptionError, "#{self.class.name} doesn't define the option: #{name}") unless option_defined?(name)
         end
       end
 

data/lib/ddtrace/configuration/settings.rb

@@ -3,8 +3,10 @@ require 'ddtrace/configuration/base'
 
 require 'ddtrace/ext/analytics'
 require 'ddtrace/ext/distributed'
+require 'ddtrace/ext/profiling'
 require 'ddtrace/ext/runtime'
 require 'ddtrace/ext/sampling'
+require 'ddtrace/ext/test'
 
 module Datadog
   module Configuration
@@ -13,9 +15,32 @@ module Datadog
     class Settings
       include Base
 
-      #
-      # Configuration options
-      #
+      def initialize(*_)
+        super
+
+        # WORKAROUND: The values for services, version, and env can get set either directly OR as a side effect of
+        # accessing tags (reading or writing). This is of course really confusing and error-prone, e.g. in an app
+        # WITHOUT this workaround where you define `DD_TAGS=env:envenvtag,service:envservicetag,version:envversiontag`
+        # and do:
+        #
+        # puts Datadog.configuration.instance_exec { "#{service} #{env} #{version}" }
+        # Datadog.configuration.tags
+        # puts Datadog.configuration.instance_exec { "#{service} #{env} #{version}" }
+        #
+        # the output will be:
+        #
+        # [empty]
+        # envservicetag envenvtag envversiontag
+        #
+        # That is -- the proper values for service/env/version are only set AFTER something accidentally or not triggers
+        # the resolution of the tags.
+        # This is really confusing, error prone, etc, so calling tags here is a really hacky but effective way to
+        # avoid this. I could not think of a better way of fixing this issue without massive refactoring of tags parsing
+        # (so that the individual service/env/version get correctly set even from their tags values, not as a side
+        # effect). Sorry :(
+        tags
+      end
+
       settings :analytics do
         option :enabled do |o|
           o.default { env_to_bool(Ext::Analytics::ENV_TRACE_ANALYTICS_ENABLED, nil) }
@@ -40,6 +65,11 @@ module Datadog
         option :debug do |o|
           o.default { env_to_bool(Datadog::Ext::Diagnostics::DD_TRACE_DEBUG, false) }
           o.lazy
+          o.on_set do |enabled|
+            # Enable rich debug print statements.
+            # We do not need to unnecessarily load 'pp' unless in debugging mode.
+            require 'pp' if enabled
+          end
         end
 
         settings :health_metrics do
@@ -64,7 +94,8 @@ module Datadog
         option :propagation_extract_style do |o|
           o.default do
             # Look for all headers by default
-            env_to_list(Ext::DistributedTracing::PROPAGATION_EXTRACT_STYLE_ENV,
+            env_to_list([Ext::DistributedTracing::PROPAGATION_STYLE_EXTRACT_ENV,
+                         Ext::DistributedTracing::PROPAGATION_EXTRACT_STYLE_ENV_OLD],
                         [Ext::DistributedTracing::PROPAGATION_STYLE_DATADOG,
                          Ext::DistributedTracing::PROPAGATION_STYLE_B3,
                          Ext::DistributedTracing::PROPAGATION_STYLE_B3_SINGLE_HEADER])
@@ -76,7 +107,8 @@ module Datadog
         option :propagation_inject_style do |o|
           o.default do
             # Only inject Datadog headers by default
-            env_to_list(Ext::DistributedTracing::PROPAGATION_INJECT_STYLE_ENV,
+            env_to_list([Ext::DistributedTracing::PROPAGATION_STYLE_INJECT_ENV,
+                         Ext::DistributedTracing::PROPAGATION_INJECT_STYLE_ENV_OLD],
                         [Ext::DistributedTracing::PROPAGATION_STYLE_DATADOG])
           end
 
@@ -85,13 +117,13 @@ module Datadog
       end
 
       option :env do |o|
+        # NOTE: env also gets set as a side effect of tags. See the WORKAROUND note in #initialize for details.
         o.default { ENV.fetch(Ext::Environment::ENV_ENVIRONMENT, nil) }
         o.lazy
       end
 
       settings :logger do
         option :instance do |o|
-          o.setter { |value, old_value| value.is_a?(::Logger) ? value : old_value }
           o.on_set { |value| set_option(:level, value.level) unless value.nil? }
         end
 
@@ -102,6 +134,49 @@ module Datadog
         get_option(:logger).instance = logger
       end
 
+      settings :profiling do
+        option :enabled do |o|
+          o.default { env_to_bool(Ext::Profiling::ENV_ENABLED, false) }
+          o.lazy
+        end
+
+        settings :exporter do
+          option :transport
+          option :transport_options do |o|
+            o.setter do
+              # NOTE: As of April 2021 there may be a few profiler private beta customers with this setting, but since I'm
+              # marking this as deprecated before public beta, we can remove this for 1.0 without concern.
+              Datadog.logger.warn(
+                'Configuring the profiler c.profiling.exporter.transport_options is no longer needed, as the profiler ' \
+                'will reuse your existing global or tracer configuration. ' \
+                'This setting is deprecated for removal in a future ddtrace version ' \
+                '(1.0 or profiling GA, whichever comes first).'
+              )
+              nil
+            end
+            o.default { nil }
+            o.lazy
+          end
+        end
+
+        option :max_events, default: 32768
+
+        # Controls the maximum number of frames for each thread sampled. Can be tuned to avoid omitted frames in the
+        # produced profiles. Increasing this may increase the overhead of profiling.
+        option :max_frames do |o|
+          o.default { env_to_int(Ext::Profiling::ENV_MAX_FRAMES, 400) }
+          o.lazy
+        end
+
+        settings :upload do
+          option :timeout_seconds do |o|
+            o.setter { |value| value.nil? ? 30.0 : value.to_f }
+            o.default { env_to_float(Ext::Profiling::ENV_UPLOAD_TIMEOUT, 30.0) }
+            o.lazy
+          end
+        end
+      end
+
       option :report_hostname do |o|
         o.default { env_to_bool(Ext::NET::ENV_REPORT_HOSTNAME, false) }
         o.lazy
@@ -150,6 +225,7 @@ module Datadog
       end
 
       option :service do |o|
+        # NOTE: service also gets set as a side effect of tags. See the WORKAROUND note in #initialize for details.
         o.default { ENV.fetch(Ext::Environment::ENV_SERVICE, nil) }
         o.lazy
       end
@@ -178,12 +254,11 @@ module Datadog
 
         o.setter do |new_value, old_value|
           # Coerce keys to strings
-          string_tags = Hash[new_value.collect { |k, v| [k.to_s, v] }]
+          string_tags = new_value.collect { |k, v| [k.to_s, v] }.to_h
 
           # Cross-populate tag values with other settings
-          if env.nil? && string_tags.key?(Ext::Environment::TAG_ENV)
-            self.env = string_tags[Ext::Environment::TAG_ENV]
-          end
+
+          self.env = string_tags[Ext::Environment::TAG_ENV] if env.nil? && string_tags.key?(Ext::Environment::TAG_ENV)
 
           if version.nil? && string_tags.key?(Ext::Environment::TAG_VERSION)
             self.version = string_tags[Ext::Environment::TAG_VERSION]
@@ -200,6 +275,39 @@ module Datadog
         o.lazy
       end
 
+      settings :test_mode do
+        option :enabled do |o|
+          o.default { env_to_bool(Ext::Test::ENV_MODE_ENABLED, false) }
+          o.lazy
+        end
+
+        option :context_flush do |o|
+          o.default { nil }
+          o.lazy
+        end
+
+        option :writer_options do |o|
+          o.default { {} }
+          o.lazy
+        end
+      end
+
+      option :time_now_provider do |o|
+        o.default { ::Time.now }
+
+        o.on_set do |time_provider|
+          Utils::Time.now_provider = time_provider
+        end
+
+        o.resetter do |_value|
+          # TODO: Resetter needs access to the default value
+          # TODO: to help reduce duplication.
+          -> { ::Time.now }.tap do |default|
+            Utils::Time.now_provider = default
+          end
+        end
+      end
+
       settings :tracer do
         option :enabled do |o|
           o.default { env_to_bool(Datadog::Ext::Diagnostics::DD_TRACE_ENABLED, true) }
@@ -271,6 +379,7 @@ module Datadog
       end
 
       option :version do |o|
+        # NOTE: version also gets set as a side effect of tags. See the WORKAROUND note in #initialize for details.
         o.default { ENV.fetch(Ext::Environment::ENV_VERSION, nil) }
         o.lazy
       end