ddtrace 0.41.0 → 0.52.0

data/lib/ddtrace/configuration/components.rb

@@ -1,13 +1,18 @@
+# typed: true
+require 'ddtrace/configuration/agent_settings_resolver'
 require 'ddtrace/diagnostics/health'
 require 'ddtrace/logger'
+require 'ddtrace/profiling'
 require 'ddtrace/runtime/metrics'
 require 'ddtrace/tracer'
+require 'ddtrace/sync_writer'
 require 'ddtrace/workers/runtime_metrics'
 
 module Datadog
   module Configuration
     # Global components for the trace library.
-    # rubocop:disable Metrics/LineLength
+    # rubocop:disable Metrics/ClassLength
+    # rubocop:disable Layout/LineLength
     class Components
       class << self
         def build_health_metrics(settings)
@@ -15,11 +20,11 @@ module Datadog
           options = { enabled: settings.enabled }
           options[:statsd] = settings.statsd unless settings.statsd.nil?
 
-          Datadog::Diagnostics::Health::Metrics.new(options)
+          Datadog::Diagnostics::Health::Metrics.new(**options)
         end
 
         def build_logger(settings)
-          logger = settings.logger.instance || Datadog::Logger.new(STDOUT)
+          logger = settings.logger.instance || Datadog::Logger.new($stdout)
           logger.level = settings.diagnostics.debug ? ::Logger::DEBUG : settings.logger.level
 
           logger
@@ -30,7 +35,7 @@ module Datadog
           options[:statsd] = settings.runtime_metrics.statsd unless settings.runtime_metrics.statsd.nil?
           options[:services] = [settings.service] unless settings.service.nil?
 
-          Datadog::Runtime::Metrics.new(options)
+          Datadog::Runtime::Metrics.new(**options)
         end
 
         def build_runtime_metrics_worker(settings)
@@ -43,7 +48,7 @@ module Datadog
           Datadog::Workers::RuntimeMetrics.new(options)
         end
 
-        def build_tracer(settings)
+        def build_tracer(settings, agent_settings)
           # If a custom tracer has been provided, use it instead.
           # Ignore all other options (they should already be configured.)
           tracer = settings.tracer.instance
@@ -53,7 +58,14 @@ module Datadog
             default_service: settings.service,
             enabled: settings.tracer.enabled,
             partial_flush: settings.tracer.partial_flush.enabled,
-            tags: build_tracer_tags(settings)
+            tags: build_tracer_tags(settings),
+            sampler: PrioritySampler.new(
+              base_sampler: AllSampler.new,
+              post_sampler: Sampling::RuleSampler.new(
+                rate_limit: settings.sampling.rate_limit,
+                default_sample_rate: settings.sampling.default_rate
+              )
+            )
           )
 
           # TODO: We reconfigure the tracer here because it has way too many
@@ -61,40 +73,143 @@ module Datadog
           #       tracer initialization for now. Just reconfigure using the
           #       existing mutable #configure function. Remove when these components
           #       are extracted.
-          tracer.configure(build_tracer_options(settings))
+          tracer.configure(agent_settings: agent_settings, **build_tracer_options(settings, agent_settings))
 
           tracer
         end
 
+        def build_profiler(settings, agent_settings, tracer)
+          return unless Datadog::Profiling.supported? && settings.profiling.enabled
+
+          unless defined?(Datadog::Profiling::Tasks::Setup)
+            # In #1545 a user reported a NameError due to this constant being uninitialized
+            # I've documented my suspicion on why that happened in
+            # https://github.com/DataDog/dd-trace-rb/issues/1545#issuecomment-856049025
+            #
+            # > Thanks for the info! It seems to feed into my theory: there's two moments in the code where we check if
+            # > profiler is "supported": 1) when loading ddtrace (inside preload) and 2) when starting the profile
+            # > after Datadog.configure gets run.
+            # > The problem is that the code assumes that both checks 1) and 2) will always reach the same conclusion:
+            # > either profiler is supported, or profiler is not supported.
+            # > In the problematic case, it looks like in your case check 1 decides that profiler is not
+            # > supported => doesn't load it, and then check 2 decides that it is => assumes it is loaded and tries to
+            # > start it.
+            #
+            # I was never able to validate if this was the issue or why exactly .supported? would change its mind BUT
+            # just in case it happens again, I've left this check which avoids breaking the user's application AND
+            # would instead direct them to report it to us instead, so that we can investigate what's wrong.
+            #
+            # TODO: As of June 2021, most checks in .supported? are related to the google-protobuf gem; so it's
+            # very likely that it was the origin of the issue we saw. Thus, if, as planned we end up moving away from
+            # protobuf OR enough time has passed and no users saw the issue again, we can remove this check altogether.
+            Datadog.logger.error(
+              'Profiling was marked as supported and enabled, but setup task was not loaded properly. ' \
+              'Please report this at https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug'
+            )
+
+            return
+          end
+
+          # Load extensions needed to support some of the Profiling features
+          Datadog::Profiling::Tasks::Setup.new.run
+
+          # NOTE: Please update the Initialization section of ProfilingDevelopment.md with any changes to this method
+
+          trace_identifiers_helper = Datadog::Profiling::TraceIdentifiers::Helper.new(
+            tracer: tracer,
+            extract_trace_resource: settings.profiling.advanced.extract_trace_resource
+          )
+
+          recorder = build_profiler_recorder(settings)
+          collectors = build_profiler_collectors(settings, recorder, trace_identifiers_helper)
+          exporters = build_profiler_exporters(settings, agent_settings)
+          scheduler = build_profiler_scheduler(settings, recorder, exporters)
+
+          Datadog::Profiler.new(collectors, scheduler)
+        end
+
         private
 
         def build_tracer_tags(settings)
           settings.tags.dup.tap do |tags|
-            tags['env'] = settings.env unless settings.env.nil?
-            tags['version'] = settings.version unless settings.version.nil?
+            tags[Ext::Environment::TAG_ENV] = settings.env unless settings.env.nil?
+            tags[Ext::Environment::TAG_VERSION] = settings.version unless settings.version.nil?
+          end
+        end
+
+        def build_tracer_options(settings, agent_settings)
+          tracer_options = {}.tap do |opts|
+            tset = settings.tracer
+            opts[:min_spans_before_partial_flush] = tset.partial_flush.min_spans_threshold unless tset.partial_flush.min_spans_threshold.nil?
+            opts[:partial_flush] = tset.partial_flush.enabled unless tset.partial_flush.enabled.nil?
+            opts[:priority_sampling] = tset.priority_sampling unless tset.priority_sampling.nil?
+            opts[:sampler] = tset.sampler unless tset.sampler.nil?
+            opts[:writer] = tset.writer unless tset.writer.nil?
+            opts[:writer_options] = tset.writer_options if tset.writer.nil?
+          end
+
+          # Apply test mode settings if test mode is activated
+          if settings.test_mode.enabled
+            build_tracer_test_mode_options(tracer_options, settings, agent_settings)
+          else
+            tracer_options
           end
         end
 
-        def build_tracer_options(settings)
-          settings = settings.tracer
-
-          {}.tap do |opts|
-            opts[:hostname] = settings.hostname unless settings.hostname.nil?
-            opts[:min_spans_before_partial_flush] = settings.partial_flush.min_spans_threshold unless settings.partial_flush.min_spans_threshold.nil?
-            opts[:partial_flush] = settings.partial_flush.enabled unless settings.partial_flush.enabled.nil?
-            opts[:port] = settings.port unless settings.port.nil?
-            opts[:priority_sampling] = settings.priority_sampling unless settings.priority_sampling.nil?
-            opts[:sampler] = settings.sampler unless settings.sampler.nil?
-            opts[:transport_options] = settings.transport_options
-            opts[:writer] = settings.writer unless settings.writer.nil?
-            opts[:writer_options] = settings.writer_options if settings.writer.nil?
+        def build_tracer_test_mode_options(tracer_options, settings, agent_settings)
+          tracer_options.tap do |opts|
+            # Do not sample any spans for tests; all must be preserved.
+            opts[:sampler] = Datadog::AllSampler.new
+
+            # If context flush behavior is provided, use it instead.
+            opts[:context_flush] = settings.test_mode.context_flush if settings.test_mode.context_flush
+
+            # Flush traces synchronously, to guarantee they are written.
+            writer_options = settings.test_mode.writer_options || {}
+            writer_options[:agent_settings] = agent_settings if agent_settings
+            opts[:writer] = Datadog::SyncWriter.new(writer_options)
           end
         end
+
+        def build_profiler_recorder(settings)
+          event_classes = [Datadog::Profiling::Events::StackSample]
+
+          Datadog::Profiling::Recorder.new(event_classes, settings.profiling.advanced.max_events)
+        end
+
+        def build_profiler_collectors(settings, recorder, trace_identifiers_helper)
+          [
+            Datadog::Profiling::Collectors::Stack.new(
+              recorder,
+              trace_identifiers_helper: trace_identifiers_helper,
+              max_frames: settings.profiling.advanced.max_frames
+              # TODO: Provide proc that identifies Datadog worker threads?
+              # ignore_thread: settings.profiling.ignore_profiler
+            )
+          ]
+        end
+
+        def build_profiler_exporters(settings, agent_settings)
+          transport =
+            settings.profiling.exporter.transport || Datadog::Profiling::Transport::HTTP.default(
+              agent_settings: agent_settings,
+              site: settings.site,
+              api_key: settings.api_key,
+              profiling_upload_timeout_seconds: settings.profiling.upload.timeout_seconds
+            )
+
+          [Datadog::Profiling::Exporter.new(transport)]
+        end
+
+        def build_profiler_scheduler(settings, recorder, exporters)
+          Datadog::Profiling::Scheduler.new(recorder, exporters)
+        end
       end
 
       attr_reader \
         :health_metrics,
         :logger,
+        :profiler,
         :runtime_metrics,
         :tracer
 
@@ -102,8 +217,13 @@ module Datadog
         # Logger
         @logger = self.class.build_logger(settings)
 
+        agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
+
         # Tracer
-        @tracer = self.class.build_tracer(settings)
+        @tracer = self.class.build_tracer(settings, agent_settings)
+
+        # Profiler
+        @profiler = self.class.build_profiler(settings, agent_settings, @tracer)
 
         # Runtime metrics
         @runtime_metrics = self.class.build_runtime_metrics_worker(settings)
@@ -113,7 +233,20 @@ module Datadog
       end
 
       # Starts up components
-      def startup!(settings); end
+      def startup!(settings)
+        if settings.profiling.enabled
+          if profiler
+            @logger.debug('Profiling started')
+            profiler.start
+          else
+            # Display a warning for users who expected profiling to be enabled
+            unsupported_reason = Datadog::Profiling.unsupported_reason
+            logger.warn("Profiling was requested but is not supported, profiling disabled: #{unsupported_reason}")
+          end
+        else
+          @logger.debug('Profiling is disabled')
+        end
+      end
 
       # Shuts down all the components in use.
       # If it has another instance to compare to, it will compare
@@ -123,12 +256,20 @@ module Datadog
         # (e.g. a custom tracer instance passed in.)
         tracer.shutdown! unless replacement && tracer == replacement.tracer
 
+        # Shutdown old profiler
+        profiler.shutdown! unless profiler.nil?
+
         # Shutdown workers
-        runtime_metrics.enabled = false
-        runtime_metrics.stop(true)
+        runtime_metrics.stop(true, close_metrics: false)
 
         # Shutdown the old metrics, unless they are still being used.
         # (e.g. custom Statsd instances.)
+        #
+        # TODO: This violates the encapsulation created by Runtime::Metrics and
+        # Health::Metrics, by directly manipulating `statsd` and changing
+        # it's lifecycle management.
+        # If we need to directly have ownership of `statsd` lifecycle, we should
+        # have direct ownership of it.
         old_statsd = [
           runtime_metrics.metrics.statsd,
           health_metrics.statsd

data/lib/ddtrace/configuration/dependency_resolver.rb

@@ -1,3 +1,4 @@
+# typed: false
 require 'tsort'
 
 module Datadog

data/lib/ddtrace/configuration/option.rb

@@ -1,3 +1,4 @@
+# typed: true
 module Datadog
   module Configuration
     # Represents an instance of an integration configuration option

data/lib/ddtrace/configuration/option_definition.rb

@@ -1,3 +1,4 @@
+# typed: true
 require 'ddtrace/configuration/option'
 
 module Datadog
@@ -60,7 +61,7 @@ module Datadog
         end
 
         def default(value = nil, &block)
-          @default = block_given? ? block : value
+          @default = block || value
         end
 
         def delegate_to(&block)
@@ -71,8 +72,6 @@ module Datadog
           @helpers[name] = block
         end
 
-        # rubocop:disable Style/TrivialAccessors
-        # (Rubocop erroneously thinks #lazy == #lazy= )
         def lazy(value = true)
           @lazy = value
         end

data/lib/ddtrace/configuration/option_definition_set.rb

@@ -1,3 +1,4 @@
+# typed: true
 require 'ddtrace/configuration/dependency_resolver'
 
 module Datadog

data/lib/ddtrace/configuration/option_set.rb

@@ -1,3 +1,4 @@
+# typed: true
 module Datadog
   module Configuration
     class OptionSet < Hash

data/lib/ddtrace/configuration/options.rb

@@ -1,3 +1,4 @@
+# typed: false
 require 'ddtrace/configuration/option_set'
 require 'ddtrace/configuration/option_definition'
 require 'ddtrace/configuration/option_definition_set'
@@ -7,17 +8,15 @@ module Datadog
     # Behavior for a configuration object that has options
     module Options
       def self.included(base)
-        base.send(:extend, ClassMethods)
-        base.send(:include, InstanceMethods)
+        base.extend(ClassMethods)
+        base.include(InstanceMethods)
       end
 
       # Class behavior for a configuration object with options
       module ClassMethods
         def options
-          @options ||= begin
-            # Allows for class inheritance of option definitions
-            superclass <= Options ? superclass.options.dup : OptionDefinitionSet.new
-          end
+          # Allows for class inheritance of option definitions
+          @options ||= superclass <= Options ? superclass.options.dup : OptionDefinitionSet.new
         end
 
         protected
@@ -51,6 +50,7 @@ module Datadog
         def define_helpers(helpers)
           helpers.each do |name, block|
             next unless block.is_a?(Proc)
+
             define_method(name, &block)
           end
         end
@@ -102,9 +102,7 @@ module Datadog
         end
 
         def assert_valid_option!(name)
-          unless option_defined?(name)
-            raise(InvalidOptionError, "#{self.class.name} doesn't define the option: #{name}")
-          end
+          raise(InvalidOptionError, "#{self.class.name} doesn't define the option: #{name}") unless option_defined?(name)
         end
       end
 

data/lib/ddtrace/configuration/pin_setup.rb

@@ -1,3 +1,4 @@
+# typed: true
 module Datadog
   module Configuration
     # PinSetup translates a flat hash into a Pin configuration

data/lib/ddtrace/configuration/settings.rb

@@ -1,10 +1,13 @@
+# typed: false
 require 'logger'
 require 'ddtrace/configuration/base'
 
 require 'ddtrace/ext/analytics'
 require 'ddtrace/ext/distributed'
-require 'ddtrace/ext/runtime'
+require 'ddtrace/ext/environment'
+require 'ddtrace/ext/profiling'
 require 'ddtrace/ext/sampling'
+require 'ddtrace/ext/test'
 
 module Datadog
   module Configuration
@@ -13,9 +16,32 @@ module Datadog
     class Settings
       include Base
 
-      #
-      # Configuration options
-      #
+      def initialize(*_)
+        super
+
+        # WORKAROUND: The values for services, version, and env can get set either directly OR as a side effect of
+        # accessing tags (reading or writing). This is of course really confusing and error-prone, e.g. in an app
+        # WITHOUT this workaround where you define `DD_TAGS=env:envenvtag,service:envservicetag,version:envversiontag`
+        # and do:
+        #
+        # puts Datadog.configuration.instance_exec { "#{service} #{env} #{version}" }
+        # Datadog.configuration.tags
+        # puts Datadog.configuration.instance_exec { "#{service} #{env} #{version}" }
+        #
+        # the output will be:
+        #
+        # [empty]
+        # envservicetag envenvtag envversiontag
+        #
+        # That is -- the proper values for service/env/version are only set AFTER something accidentally or not triggers
+        # the resolution of the tags.
+        # This is really confusing, error prone, etc, so calling tags here is a really hacky but effective way to
+        # avoid this. I could not think of a better way of fixing this issue without massive refactoring of tags parsing
+        # (so that the individual service/env/version get correctly set even from their tags values, not as a side
+        # effect). Sorry :(
+        tags
+      end
+
       settings :analytics do
         option :enabled do |o|
           o.default { env_to_bool(Ext::Analytics::ENV_TRACE_ANALYTICS_ENABLED, nil) }
@@ -69,7 +95,8 @@ module Datadog
         option :propagation_extract_style do |o|
           o.default do
             # Look for all headers by default
-            env_to_list(Ext::DistributedTracing::PROPAGATION_EXTRACT_STYLE_ENV,
+            env_to_list([Ext::DistributedTracing::PROPAGATION_STYLE_EXTRACT_ENV,
+                         Ext::DistributedTracing::PROPAGATION_EXTRACT_STYLE_ENV_OLD],
                         [Ext::DistributedTracing::PROPAGATION_STYLE_DATADOG,
                          Ext::DistributedTracing::PROPAGATION_STYLE_B3,
                          Ext::DistributedTracing::PROPAGATION_STYLE_B3_SINGLE_HEADER])
@@ -81,7 +108,8 @@ module Datadog
         option :propagation_inject_style do |o|
           o.default do
             # Only inject Datadog headers by default
-            env_to_list(Ext::DistributedTracing::PROPAGATION_INJECT_STYLE_ENV,
+            env_to_list([Ext::DistributedTracing::PROPAGATION_STYLE_INJECT_ENV,
+                         Ext::DistributedTracing::PROPAGATION_INJECT_STYLE_ENV_OLD],
                         [Ext::DistributedTracing::PROPAGATION_STYLE_DATADOG])
           end
 
@@ -90,6 +118,7 @@ module Datadog
       end
 
       option :env do |o|
+        # NOTE: env also gets set as a side effect of tags. See the WORKAROUND note in #initialize for details.
         o.default { ENV.fetch(Ext::Environment::ENV_ENVIRONMENT, nil) }
         o.lazy
       end
@@ -106,6 +135,55 @@ module Datadog
         get_option(:logger).instance = logger
       end
 
+      settings :profiling do
+        option :enabled do |o|
+          o.default { env_to_bool(Ext::Profiling::ENV_ENABLED, false) }
+          o.lazy
+        end
+
+        settings :exporter do
+          option :transport
+          option :transport_options do |o|
+            o.setter do
+              # NOTE: As of April 2021 there may be a few profiler private beta customers with this setting, but since I'm
+              # marking this as deprecated before public beta, we can remove this for 1.0 without concern.
+              Datadog.logger.warn(
+                'Configuring the profiler c.profiling.exporter.transport_options is no longer needed, as the profiler ' \
+                'will reuse your existing global or tracer configuration. ' \
+                'This setting is deprecated for removal in a future ddtrace version ' \
+                '(1.0 or profiling GA, whichever comes first).'
+              )
+              nil
+            end
+            o.default { nil }
+            o.lazy
+          end
+        end
+
+        settings :advanced do
+          option :max_events, default: 32768
+
+          # Controls the maximum number of frames for each thread sampled. Can be tuned to avoid omitted frames in the
+          # produced profiles. Increasing this may increase the overhead of profiling.
+          option :max_frames do |o|
+            o.default { env_to_int(Ext::Profiling::ENV_MAX_FRAMES, 400) }
+            o.lazy
+          end
+
+          # When using profiling together with tracing, this controls if trace resources (usually the endpoint names)
+          # are gathered and reported together with profiles.
+          option :extract_trace_resource, default: true
+        end
+
+        settings :upload do
+          option :timeout_seconds do |o|
+            o.setter { |value| value.nil? ? 30.0 : value.to_f }
+            o.default { env_to_float(Ext::Profiling::ENV_UPLOAD_TIMEOUT, 30.0) }
+            o.lazy
+          end
+        end
+      end
+
       option :report_hostname do |o|
         o.default { env_to_bool(Ext::NET::ENV_REPORT_HOSTNAME, false) }
         o.lazy
@@ -154,8 +232,17 @@ module Datadog
       end
 
       option :service do |o|
-        o.default { ENV.fetch(Ext::Environment::ENV_SERVICE, nil) }
+        # NOTE: service also gets set as a side effect of tags. See the WORKAROUND note in #initialize for details.
+        o.default { ENV.fetch(Ext::Environment::ENV_SERVICE, Ext::Environment::FALLBACK_SERVICE_NAME) }
         o.lazy
+
+        # There's a few cases where we don't want to use the fallback service name, so this helper allows us to get a
+        # nil instead so that one can do
+        # nice_service_name = Datadog.configure.service_without_fallback || nice_service_name_default
+        o.helper(:service_without_fallback) do
+          service_name = service
+          service_name unless service_name.equal?(Ext::Environment::FALLBACK_SERVICE_NAME)
+        end
       end
 
       option :site do |o|
@@ -182,18 +269,17 @@ module Datadog
 
         o.setter do |new_value, old_value|
           # Coerce keys to strings
-          string_tags = Hash[new_value.collect { |k, v| [k.to_s, v] }]
+          string_tags = new_value.collect { |k, v| [k.to_s, v] }.to_h
 
           # Cross-populate tag values with other settings
-          if env.nil? && string_tags.key?(Ext::Environment::TAG_ENV)
-            self.env = string_tags[Ext::Environment::TAG_ENV]
-          end
+
+          self.env = string_tags[Ext::Environment::TAG_ENV] if env.nil? && string_tags.key?(Ext::Environment::TAG_ENV)
 
           if version.nil? && string_tags.key?(Ext::Environment::TAG_VERSION)
             self.version = string_tags[Ext::Environment::TAG_VERSION]
           end
 
-          if service.nil? && string_tags.key?(Ext::Environment::TAG_SERVICE)
+          if service_without_fallback.nil? && string_tags.key?(Ext::Environment::TAG_SERVICE)
             self.service = string_tags[Ext::Environment::TAG_SERVICE]
           end
 
@@ -204,6 +290,39 @@ module Datadog
         o.lazy
       end
 
+      settings :test_mode do
+        option :enabled do |o|
+          o.default { env_to_bool(Ext::Test::ENV_MODE_ENABLED, false) }
+          o.lazy
+        end
+
+        option :context_flush do |o|
+          o.default { nil }
+          o.lazy
+        end
+
+        option :writer_options do |o|
+          o.default { {} }
+          o.lazy
+        end
+      end
+
+      option :time_now_provider do |o|
+        o.default { ::Time.now }
+
+        o.on_set do |time_provider|
+          Utils::Time.now_provider = time_provider
+        end
+
+        o.resetter do |_value|
+          # TODO: Resetter needs access to the default value
+          # TODO: to help reduce duplication.
+          -> { ::Time.now }.tap do |default|
+            Utils::Time.now_provider = default
+          end
+        end
+      end
+
       settings :tracer do
         option :enabled do |o|
           o.default { env_to_bool(Datadog::Ext::Diagnostics::DD_TRACE_ENABLED, true) }
@@ -275,6 +394,7 @@ module Datadog
       end
 
       option :version do |o|
+        # NOTE: version also gets set as a side effect of tags. See the WORKAROUND note in #initialize for details.
         o.default { ENV.fetch(Ext::Environment::ENV_VERSION, nil) }
         o.lazy
       end