ddtrace 1.4.2 → 1.23.3

data/lib/datadog/core/configuration/components.rb

@@ -1,24 +1,25 @@
-# typed: false
-
 require_relative 'agent_settings_resolver'
+require_relative 'ext'
 require_relative '../diagnostics/environment_logger'
 require_relative '../diagnostics/health'
 require_relative '../logger'
 require_relative '../runtime/metrics'
-require_relative '../telemetry/client'
+require_relative '../telemetry/component'
 require_relative '../workers/runtime_metrics'
 
-require_relative '../../tracing/tracer'
-require_relative '../../tracing/flush'
-require_relative '../../tracing/sync_writer'
+require_relative '../remote/component'
+require_relative '../../tracing/component'
+require_relative '../../profiling/component'
+require_relative '../../appsec/component'
 
 module Datadog
   module Core
     module Configuration
       # Global components for the trace library.
-      # rubocop:disable Metrics/ClassLength
       class Components
         class << self
+          include Datadog::Tracing::Component
+
           def build_health_metrics(settings)
             settings = settings.diagnostics.health_metrics
             options = { enabled: settings.enabled }
@@ -52,338 +53,83 @@ module Datadog
             Core::Workers::RuntimeMetrics.new(options)
           end
 
-          def build_telemetry(settings)
-            Telemetry::Client.new(enabled: settings.telemetry.enabled)
-          end
-
-          def build_tracer(settings, agent_settings)
-            # If a custom tracer has been provided, use it instead.
-            # Ignore all other options (they should already be configured.)
-            tracer = settings.tracing.instance
-            return tracer unless tracer.nil?
-
-            # Apply test mode settings if test mode is activated
-            if settings.tracing.test_mode.enabled
-              trace_flush = build_test_mode_trace_flush(settings)
-              sampler = build_test_mode_sampler
-              writer = build_test_mode_writer(settings, agent_settings)
-            else
-              trace_flush = build_trace_flush(settings)
-              sampler = build_sampler(settings)
-              writer = build_writer(settings, agent_settings)
+          def build_telemetry(settings, agent_settings, logger)
+            enabled = settings.telemetry.enabled
+            if agent_settings.adapter != Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
+              enabled = false
+              logger.debug { "Telemetry disabled. Agent network adapter not supported: #{agent_settings.adapter}" }
             end
 
-            subscribe_to_writer_events!(writer, sampler, settings.tracing.test_mode.enabled)
-
-            Tracing::Tracer.new(
-              default_service: settings.service,
-              enabled: settings.tracing.enabled,
-              trace_flush: trace_flush,
-              sampler: sampler,
-              writer: writer,
-              tags: build_tracer_tags(settings),
+            Telemetry::Component.new(
+              enabled: enabled,
+              heartbeat_interval_seconds: settings.telemetry.heartbeat_interval_seconds,
+              dependency_collection: settings.telemetry.dependency_collection
             )
           end
-
-          def build_trace_flush(settings)
-            if settings.tracing.partial_flush.enabled
-              Tracing::Flush::Partial.new(
-                min_spans_before_partial_flush: settings.tracing.partial_flush.min_spans_threshold
-              )
-            else
-              Tracing::Flush::Finished.new
-            end
-          end
-
-          # TODO: Sampler should be a top-level component.
-          # It is currently part of the Tracer initialization
-          # process, but can take a variety of options (including
-          # a fully custom instance) that makes the Tracer
-          # initialization process complex.
-          def build_sampler(settings)
-            if (sampler = settings.tracing.sampler)
-              if settings.tracing.priority_sampling == false
-                sampler
-              else
-                ensure_priority_sampling(sampler, settings)
-              end
-            elsif settings.tracing.priority_sampling == false
-              Tracing::Sampling::RuleSampler.new(
-                rate_limit: settings.tracing.sampling.rate_limit,
-                default_sample_rate: settings.tracing.sampling.default_rate
-              )
-            else
-              Tracing::Sampling::PrioritySampler.new(
-                base_sampler: Tracing::Sampling::AllSampler.new,
-                post_sampler: Tracing::Sampling::RuleSampler.new(
-                  rate_limit: settings.tracing.sampling.rate_limit,
-                  default_sample_rate: settings.tracing.sampling.default_rate
-                )
-              )
-            end
-          end
-
-          def ensure_priority_sampling(sampler, settings)
-            if sampler.is_a?(Tracing::Sampling::PrioritySampler)
-              sampler
-            else
-              Tracing::Sampling::PrioritySampler.new(
-                base_sampler: sampler,
-                post_sampler: Tracing::Sampling::RuleSampler.new(
-                  rate_limit: settings.tracing.sampling.rate_limit,
-                  default_sample_rate: settings.tracing.sampling.default_rate
-                )
-              )
-            end
-          end
-
-          # TODO: Writer should be a top-level component.
-          # It is currently part of the Tracer initialization
-          # process, but can take a variety of options (including
-          # a fully custom instance) that makes the Tracer
-          # initialization process complex.
-          def build_writer(settings, agent_settings)
-            if (writer = settings.tracing.writer)
-              return writer
-            end
-
-            Tracing::Writer.new(agent_settings: agent_settings, **settings.tracing.writer_options)
-          end
-
-          def subscribe_to_writer_events!(writer, sampler, test_mode)
-            return unless writer.respond_to?(:events) # Check if it's a custom, external writer
-
-            writer.events.after_send.subscribe(&WRITER_RECORD_ENVIRONMENT_INFORMATION_CALLBACK)
-
-            return unless sampler.is_a?(Tracing::Sampling::PrioritySampler)
-
-            # DEV: We need to ignore priority sampling updates coming from the agent in test mode
-            # because test mode wants to *unconditionally* sample all traces.
-            #
-            # This can cause trace metrics to be overestimated, but that's a trade-off we take
-            # here to achieve 100% sampling rate.
-            return if test_mode
-
-            writer.events.after_send.subscribe(&writer_update_priority_sampler_rates_callback(sampler))
-          end
-
-          WRITER_RECORD_ENVIRONMENT_INFORMATION_CALLBACK = lambda do |_, responses|
-            Core::Diagnostics::EnvironmentLogger.log!(responses)
-          end
-
-          # Create new lambda for writer callback,
-          # capture the current sampler in the callback closure.
-          def writer_update_priority_sampler_rates_callback(sampler)
-            lambda do |_, responses|
-              response = responses.last
-
-              next unless response && !response.internal_error? && response.service_rates
-
-              sampler.update(response.service_rates)
-            end
-          end
-
-          def build_profiler(settings, agent_settings, tracer)
-            return unless settings.profiling.enabled
-
-            # Workaround for weird dependency direction: the Core::Configuration::Components class currently has a
-            # dependency on individual products, in this case the Profiler.
-            # (Note "currently": in the future we want to change this so core classes don't depend on specific products)
-            #
-            # If the current file included a `require 'datadog/profiler'` at its beginning, we would generate circular
-            # requires when used from profiling:
-            #
-            # datadog/profiling
-            #     └─requires─> datadog/core
-            #                      └─requires─> datadog/core/configuration/components
-            #                                       └─requires─> datadog/profiling       # Loop!
-            #
-            # ...thus in #1998 we removed such a require.
-            #
-            # On the other hand, if datadog/core is loaded by a different product and no general `require 'ddtrace'` is
-            # done, then profiling may not be loaded, and thus to avoid this issue we do a require here (which is a
-            # no-op if profiling is already loaded).
-            require_relative '../../profiling'
-            return unless Profiling.supported?
-
-            unless defined?(Profiling::Tasks::Setup)
-              # In #1545 a user reported a NameError due to this constant being uninitialized
-              # I've documented my suspicion on why that happened in
-              # https://github.com/DataDog/dd-trace-rb/issues/1545#issuecomment-856049025
-              #
-              # > Thanks for the info! It seems to feed into my theory: there's two moments in the code where we check if
-              # > profiler is "supported": 1) when loading ddtrace (inside preload) and 2) when starting the profile
-              # > after Datadog.configure gets run.
-              # > The problem is that the code assumes that both checks 1) and 2) will always reach the same conclusion:
-              # > either profiler is supported, or profiler is not supported.
-              # > In the problematic case, it looks like in your case check 1 decides that profiler is not
-              # > supported => doesn't load it, and then check 2 decides that it is => assumes it is loaded and tries to
-              # > start it.
-              #
-              # I was never able to validate if this was the issue or why exactly .supported? would change its mind BUT
-              # just in case it happens again, I've left this check which avoids breaking the user's application AND
-              # would instead direct them to report it to us instead, so that we can investigate what's wrong.
-              #
-              # TODO: As of June 2021, most checks in .supported? are related to the google-protobuf gem; so it's
-              # very likely that it was the origin of the issue we saw. Thus, if, as planned we end up moving away from
-              # protobuf OR enough time has passed and no users saw the issue again, we can remove this check altogether.
-              Datadog.logger.error(
-                'Profiling was marked as supported and enabled, but setup task was not loaded properly. ' \
-                'Please report this at https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug'
-              )
-
-              return
-            end
-
-            # Load extensions needed to support some of the Profiling features
-            Profiling::Tasks::Setup.new.run
-
-            # NOTE: Please update the Initialization section of ProfilingDevelopment.md with any changes to this method
-
-            if settings.profiling.advanced.force_enable_new_profiler
-              recorder = Datadog::Profiling::StackRecorder.new
-              collector = Datadog::Profiling::Collectors::CpuAndWallTimeWorker.new(
-                recorder: recorder,
-                max_frames: settings.profiling.advanced.max_frames
-              )
-            else
-              trace_identifiers_helper = Profiling::TraceIdentifiers::Helper.new(
-                tracer: tracer,
-                endpoint_collection_enabled: settings.profiling.advanced.endpoint.collection.enabled
-              )
-
-              recorder = build_profiler_old_recorder(settings)
-              collector = build_profiler_oldstack_collector(settings, recorder, trace_identifiers_helper)
-            end
-
-            exporter = build_profiler_exporter(settings, recorder)
-            transport = build_profiler_transport(settings, agent_settings)
-            scheduler = Profiling::Scheduler.new(exporter: exporter, transport: transport)
-
-            Profiling::Profiler.new([collector], scheduler)
-          end
-
-          private
-
-          def build_tracer_tags(settings)
-            settings.tags.dup.tap do |tags|
-              tags[Core::Environment::Ext::TAG_ENV] = settings.env unless settings.env.nil?
-              tags[Core::Environment::Ext::TAG_VERSION] = settings.version unless settings.version.nil?
-            end
-          end
-
-          def build_test_mode_trace_flush(settings)
-            # If context flush behavior is provided, use it instead.
-            settings.tracing.test_mode.trace_flush || build_trace_flush(settings)
-          end
-
-          def build_test_mode_sampler
-            # Do not sample any spans for tests; all must be preserved.
-            # Set priority sampler to ensure the agent doesn't drop any traces.
-            Tracing::Sampling::PrioritySampler.new(
-              base_sampler: Tracing::Sampling::AllSampler.new,
-              post_sampler: Tracing::Sampling::AllSampler.new
-            )
-          end
-
-          def build_test_mode_writer(settings, agent_settings)
-            # Flush traces synchronously, to guarantee they are written.
-            writer_options = settings.tracing.test_mode.writer_options || {}
-            Tracing::SyncWriter.new(agent_settings: agent_settings, **writer_options)
-          end
-
-          def build_profiler_old_recorder(settings)
-            Profiling::OldRecorder.new([Profiling::Events::StackSample], settings.profiling.advanced.max_events)
-          end
-
-          def build_profiler_exporter(settings, recorder)
-            code_provenance_collector =
-              (Profiling::Collectors::CodeProvenance.new if settings.profiling.advanced.code_provenance_enabled)
-
-            Profiling::Exporter.new(pprof_recorder: recorder, code_provenance_collector: code_provenance_collector)
-          end
-
-          def build_profiler_oldstack_collector(settings, old_recorder, trace_identifiers_helper)
-            Profiling::Collectors::OldStack.new(
-              old_recorder,
-              trace_identifiers_helper: trace_identifiers_helper,
-              max_frames: settings.profiling.advanced.max_frames
-            )
-          end
-
-          def build_profiler_transport(settings, agent_settings)
-            settings.profiling.exporter.transport ||
-              if settings.profiling.advanced.legacy_transport_enabled
-                require_relative '../../profiling/transport/http'
-
-                Datadog.logger.warn('Using legacy profiling transport. Do not use unless instructed to by support.')
-
-                Profiling::Transport::HTTP.default(
-                  agent_settings: agent_settings,
-                  site: settings.site,
-                  api_key: settings.api_key,
-                  profiling_upload_timeout_seconds: settings.profiling.upload.timeout_seconds
-                )
-              end ||
-              Profiling::HttpTransport.new(
-                agent_settings: agent_settings,
-                site: settings.site,
-                api_key: settings.api_key,
-                upload_timeout_seconds: settings.profiling.upload.timeout_seconds,
-              )
-          end
         end
 
+        include Datadog::Tracing::Component::InstanceMethods
+
         attr_reader \
           :health_metrics,
           :logger,
+          :remote,
           :profiler,
           :runtime_metrics,
           :telemetry,
-          :tracer
+          :tracer,
+          :appsec
 
         def initialize(settings)
-          # Logger
           @logger = self.class.build_logger(settings)
+          @environment_logger_extra = {}
 
+          # This agent_settings is intended for use within Core. If you require
+          # agent_settings within a product outside of core you should extend
+          # the Core resolver from within your product/component's namespace.
           agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
 
-          # Tracer
-          @tracer = self.class.build_tracer(settings, agent_settings)
+          @remote = Remote::Component.build(settings, agent_settings)
+          @tracer = self.class.build_tracer(settings, logger: @logger)
 
-          # Profiler
-          @profiler = self.class.build_profiler(settings, agent_settings, @tracer)
+          @profiler, profiler_logger_extra = Datadog::Profiling::Component.build_profiler_component(
+            settings: settings,
+            agent_settings: agent_settings,
+            optional_tracer: @tracer,
+          )
+          @environment_logger_extra.merge!(profiler_logger_extra) if profiler_logger_extra
 
-          # Runtime metrics
           @runtime_metrics = self.class.build_runtime_metrics_worker(settings)
-
-          # Health metrics
           @health_metrics = self.class.build_health_metrics(settings)
-
-          # Telemetry
-          @telemetry = self.class.build_telemetry(settings)
+          @telemetry = self.class.build_telemetry(settings, agent_settings, logger)
+          @appsec = Datadog::AppSec::Component.build_appsec_component(settings)
         end
 
         # Starts up components
         def startup!(settings)
           if settings.profiling.enabled
             if profiler
-              @logger.debug('Profiling started')
               profiler.start
             else
               # Display a warning for users who expected profiling to be enabled
               unsupported_reason = Profiling.unsupported_reason
               logger.warn("Profiling was requested but is not supported, profiling disabled: #{unsupported_reason}")
             end
-          else
-            @logger.debug('Profiling is disabled')
           end
+
+          Core::Diagnostics::EnvironmentLogger.collect_and_log!(@environment_logger_extra)
         end
 
         # Shuts down all the components in use.
         # If it has another instance to compare to, it will compare
         # and avoid tearing down parts still in use.
         def shutdown!(replacement = nil)
+          # Shutdown remote configuration
+          remote.shutdown! if remote
+
+          # Decommission AppSec
+          appsec.shutdown! if appsec
+
           # Shutdown the old tracer, unless it's still being used.
           # (e.g. a custom tracer instance passed in.)
           tracer.shutdown! unless replacement && tracer == replacement.tracer
@@ -419,11 +165,11 @@ module Datadog
           unused_statsd = (old_statsd - (old_statsd & new_statsd))
           unused_statsd.each(&:close)
 
-          telemetry.stop!
+          # enqueue closing event before stopping telemetry so it will be send out on shutdown
           telemetry.emit_closing! unless replacement
+          telemetry.stop!
         end
       end
-      # rubocop:enable Metrics/ClassLength
     end
   end
 end

data/lib/datadog/core/configuration/ext.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Configuration
+      # Constants for configuration settings
+      # e.g. Env vars, default values, enums, etc...
+      module Ext
+        # @public_api
+        module Diagnostics
+          ENV_DEBUG_ENABLED = 'DD_TRACE_DEBUG'
+          ENV_HEALTH_METRICS_ENABLED = 'DD_HEALTH_METRICS_ENABLED'
+          ENV_STARTUP_LOGS_ENABLED = 'DD_TRACE_STARTUP_LOGS'
+        end
+
+        module Metrics
+          ENV_DEFAULT_PORT = 'DD_METRIC_AGENT_PORT'
+        end
+
+        # DEV-2.0: This module only exists for backwards compatibility with the public API.
+        # It should be consolidated into the Agent module below.
+        module Transport
+          ENV_DEFAULT_HOST = 'DD_AGENT_HOST'
+        end
+
+        module Agent
+          # env var has "trace" in it, but it really applies to all products
+          ENV_DEFAULT_PORT = 'DD_TRACE_AGENT_PORT'
+          ENV_DEFAULT_URL = 'DD_TRACE_AGENT_URL'
+
+          module HTTP
+            ADAPTER = :net_http # DEV: Rename to simply `:http`, as Net::HTTP is an implementation detail.
+            DEFAULT_HOST = '127.0.0.1'
+            DEFAULT_PORT = 8126
+          end
+
+          # @public_api
+          module UnixSocket
+            ADAPTER = :unix
+            DEFAULT_PATH = '/var/run/datadog/apm.socket'
+            DEFAULT_TIMEOUT_SECONDS = 1
+          end
+        end
+      end
+    end
+  end
+end