ddtrace 1.3.0 → 1.4.1

data/lib/datadog/core/configuration/base.rb

@@ -79,6 +79,15 @@ module Datadog
             options_hash
           end
 
+          # Retrieves a nested option from a list of symbols
+          def dig(*options)
+            raise ArgumentError, 'expected at least one option' if options.empty?
+
+            options.inject(self) do |receiver, option|
+              receiver.send(option)
+            end
+          end
+
           def reset!
             reset_options!
           end

data/lib/datadog/core/configuration/components.rb

@@ -5,6 +5,7 @@ require_relative '../diagnostics/environment_logger'
 require_relative '../diagnostics/health'
 require_relative '../logger'
 require_relative '../runtime/metrics'
+require_relative '../telemetry/client'
 require_relative '../workers/runtime_metrics'
 
 require_relative '../../tracing/tracer'
@@ -51,6 +52,10 @@ module Datadog
             Core::Workers::RuntimeMetrics.new(options)
           end
 
+          def build_telemetry(settings)
+            Telemetry::Client.new(enabled: settings.telemetry.enabled)
+          end
+
           def build_tracer(settings, agent_settings)
             # If a custom tracer has been provided, use it instead.
             # Ignore all other options (they should already be configured.)
@@ -235,13 +240,21 @@ module Datadog
 
             # NOTE: Please update the Initialization section of ProfilingDevelopment.md with any changes to this method
 
-            trace_identifiers_helper = Profiling::TraceIdentifiers::Helper.new(
-              tracer: tracer,
-              endpoint_collection_enabled: settings.profiling.advanced.endpoint.collection.enabled
-            )
+            if settings.profiling.advanced.force_enable_new_profiler
+              recorder = Datadog::Profiling::StackRecorder.new
+              collector = Datadog::Profiling::Collectors::CpuAndWallTimeWorker.new(
+                recorder: recorder,
+                max_frames: settings.profiling.advanced.max_frames
+              )
+            else
+              trace_identifiers_helper = Profiling::TraceIdentifiers::Helper.new(
+                tracer: tracer,
+                endpoint_collection_enabled: settings.profiling.advanced.endpoint.collection.enabled
+              )
 
-            recorder = build_profiler_old_recorder(settings)
-            collector = build_profiler_oldstack_collector(settings, recorder, trace_identifiers_helper)
+              recorder = build_profiler_old_recorder(settings)
+              collector = build_profiler_oldstack_collector(settings, recorder, trace_identifiers_helper)
+            end
 
             exporter = build_profiler_exporter(settings, recorder)
             transport = build_profiler_transport(settings, agent_settings)
@@ -326,6 +339,7 @@ module Datadog
           :logger,
           :profiler,
           :runtime_metrics,
+          :telemetry,
           :tracer
 
         def initialize(settings)
@@ -345,6 +359,9 @@ module Datadog
 
           # Health metrics
           @health_metrics = self.class.build_health_metrics(settings)
+
+          # Telemetry
+          @telemetry = self.class.build_telemetry(settings)
         end
 
         # Starts up components
@@ -401,6 +418,9 @@ module Datadog
 
           unused_statsd = (old_statsd - (old_statsd & new_statsd))
           unused_statsd.each(&:close)
+
+          telemetry.stop!
+          telemetry.emit_closing! unless replacement
         end
       end
       # rubocop:enable Metrics/ClassLength

data/lib/datadog/core/configuration/settings.rb

@@ -5,6 +5,7 @@ require 'logger'
 require_relative 'base'
 require_relative '../environment/ext'
 require_relative '../runtime/ext'
+require_relative '../telemetry/ext'
 require_relative '../../profiling/ext'
 require_relative '../../tracing/configuration/ext'
 
@@ -240,6 +241,16 @@ module Datadog
               o.default { env_to_bool('DD_PROFILING_LEGACY_TRANSPORT_ENABLED', false) }
               o.lazy
             end
+
+            # Forces enabling the new profiler. We do not yet recommend turning on this option.
+            #
+            # Note that setting this to "false" (or not setting it) will not prevent the new profiler from
+            # being automatically used in the future.
+            # This option will be deprecated for removal once the new profiler gets enabled by default for all customers.
+            option :force_enable_new_profiler do |o|
+              o.default { env_to_bool('DD_PROFILING_FORCE_ENABLE_NEW', false) }
+              o.lazy
+            end
           end
 
           # @public_api
@@ -618,6 +629,20 @@ module Datadog
           o.default { ENV.fetch(Core::Environment::Ext::ENV_VERSION, nil) }
           o.lazy
         end
+
+        # Client-side telemetry configuration
+        # @public_api
+        settings :telemetry do
+          # Enable telemetry collection. This allows telemetry events to be emitted to the telemetry API.
+          #
+          # @default `DD_INSTRUMENTATION_TELEMETRY_ENABLED` environment variable, otherwise `false`. In a future release,
+          #   this value will be changed to `true` by default as documented [here](https://docs.datadoghq.com/tracing/configure_data_security/#telemetry-collection).
+          # @return [Boolean]
+          option :enabled do |o|
+            o.default { env_to_bool(Core::Telemetry::Ext::ENV_ENABLED, false) }
+            o.lazy
+          end
+        end
       end
       # rubocop:enable Metrics/BlockLength
       # rubocop:enable Metrics/ClassLength

data/lib/datadog/core/configuration.rb

@@ -2,6 +2,7 @@
 
 require_relative 'configuration/components'
 require_relative 'configuration/settings'
+require_relative 'telemetry/emitter'
 require_relative 'logger'
 require_relative 'pin'
 
@@ -89,7 +90,9 @@ module Datadog
             if components?
               replace_components!(configuration, @components)
             else
-              build_components(configuration)
+              components = build_components(configuration)
+              components.telemetry.started!
+              components
             end
           )
         end

data/lib/datadog/core/telemetry/client.rb

@@ -0,0 +1,79 @@
+# typed: true
+
+require_relative 'emitter'
+require_relative 'heartbeat'
+require_relative '../utils/forking'
+
+module Datadog
+  module Core
+    module Telemetry
+      # Telemetry entrypoint, coordinates sending telemetry events at various points in app lifecyle
+      class Client
+        attr_reader \
+          :emitter,
+          :enabled,
+          :unsupported,
+          :worker
+
+        include Core::Utils::Forking
+
+        # @param enabled [Boolean] Determines whether telemetry events should be sent to the API
+        def initialize(enabled: true)
+          @enabled = enabled
+          @emitter = Emitter.new
+          @stopped = false
+          @unsupported = false
+          @worker = Telemetry::Heartbeat.new(enabled: @enabled) do
+            heartbeat!
+          end
+        end
+
+        def disable!
+          @enabled = false
+          @worker.enabled = false
+        end
+
+        def started!
+          return if !@enabled || forked?
+
+          res = @emitter.request(:'app-started')
+
+          if res.not_found? # Telemetry is only supported by agent versions 7.34 and up
+            Datadog.logger.debug('Agent does not support telemetry; disabling future telemetry events.')
+            disable!
+            @unsupported = true # Prevent telemetry from getting re-enabled
+          end
+
+          res
+        end
+
+        def emit_closing!
+          return if !@enabled || forked?
+
+          @emitter.request(:'app-closing')
+        end
+
+        def stop!
+          return if @stopped
+
+          @worker.stop(true, 0)
+          @stopped = true
+        end
+
+        def integrations_change!
+          return if !@enabled || forked?
+
+          @emitter.request(:'app-integrations-change')
+        end
+
+        private
+
+        def heartbeat!
+          return if !@enabled || forked?
+
+          @emitter.request(:'app-heartbeat')
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/collector.rb

@@ -0,0 +1,234 @@
+# frozen_string_literal: true
+
+# typed: true
+
+require 'etc'
+
+require_relative '../configuration/agent_settings_resolver'
+require_relative '../environment/ext'
+require_relative '../environment/platform'
+require_relative 'v1/application'
+require_relative 'v1/dependency'
+require_relative 'v1/host'
+require_relative 'v1/integration'
+require_relative 'v1/product'
+require_relative '../../../ddtrace/transport/ext'
+
+module Datadog
+  module Core
+    module Telemetry
+      # Module defining methods for collecting metadata for telemetry
+      # rubocop:disable Metrics/ModuleLength
+      module Collector
+        include Datadog::Core::Configuration
+
+        # Forms a hash of configuration key value pairs to be sent in the additional payload
+        def additional_payload
+          additional_payload_variables
+        end
+
+        # Forms a telemetry application object
+        def application
+          Telemetry::V1::Application.new(
+            env: env,
+            language_name: Datadog::Core::Environment::Ext::LANG,
+            language_version: Datadog::Core::Environment::Ext::LANG_VERSION,
+            products: products,
+            runtime_name: Datadog::Core::Environment::Ext::RUBY_ENGINE,
+            runtime_version: Datadog::Core::Environment::Ext::ENGINE_VERSION,
+            service_name: service_name,
+            service_version: service_version,
+            tracer_version: library_version
+          )
+        end
+
+        # Forms a hash of standard key value pairs to be sent in the app-started event configuration
+        def configurations
+          configurations = {
+            DD_AGENT_HOST: Datadog.configuration.agent.host,
+            DD_AGENT_TRANSPORT: agent_transport,
+            DD_TRACE_SAMPLE_RATE: format_configuration_value(Datadog.configuration.tracing.sampling.default_rate),
+          }
+          compact_hash(configurations)
+        end
+
+        # Forms a telemetry app-started dependencies object
+        def dependencies
+          Gem.loaded_specs.collect do |name, loaded_gem|
+            Datadog::Core::Telemetry::V1::Dependency.new(
+              name: name, version: loaded_gem.version.to_s, hash: loaded_gem.hash.to_s
+            )
+          end
+        end
+
+        # Forms a telemetry host object
+        def host
+          Telemetry::V1::Host.new(
+            container_id: Core::Environment::Container.container_id,
+            hostname: Core::Environment::Platform.hostname,
+            kernel_name: Core::Environment::Platform.kernel_name,
+            kernel_release: Core::Environment::Platform.kernel_release,
+            kernel_version: Core::Environment::Platform.kernel_version
+          )
+        end
+
+        # Forms a telemetry app-started integrations object
+        def integrations
+          Datadog.registry.map do |integration|
+            is_instrumented = instrumented?(integration)
+            is_enabled = is_instrumented && patched?(integration)
+            Telemetry::V1::Integration.new(
+              name: integration.name.to_s,
+              enabled: is_enabled,
+              version: integration_version(integration),
+              compatible: integration_compatible?(integration),
+              error: (patch_error(integration) if is_instrumented && !is_enabled),
+              auto_enabled: is_enabled ? integration_auto_instrument?(integration) : nil
+            )
+          end
+        end
+
+        # Returns the runtime ID of the current process
+        def runtime_id
+          Datadog::Core::Environment::Identity.id
+        end
+
+        # Returns the current as a UNIX timestamp in seconds
+        def tracer_time
+          Time.now.to_i
+        end
+
+        private
+
+        TARGET_OPTIONS = [
+          'ci.enabled',
+          'logger.level',
+          'profiling.advanced.code_provenance_enabled',
+          'profiling.advanced.endpoint.collection.enabled',
+          'profiling.enabled',
+          'runtime_metrics.enabled',
+          'tracing.analytics.enabled',
+          'tracing.distributed_tracing.propagation_inject_style',
+          'tracing.distributed_tracing.propagation_extract_style',
+          'tracing.enabled',
+          'tracing.log_injection',
+          'tracing.partial_flush.enabled',
+          'tracing.partial_flush.min_spans_threshold',
+          'tracing.priority_sampling',
+          'tracing.report_hostname',
+          'tracing.sampling.default_rate',
+          'tracing.sampling.rate_limit'
+        ].freeze
+
+        def additional_payload_variables
+          # Whitelist of configuration options to send in additional payload object
+          configuration = Datadog.configuration
+          options = TARGET_OPTIONS.each_with_object({}) do |option, hash|
+            split_option = option.split('.')
+            hash[option] = format_configuration_value(configuration.dig(*split_option))
+          end
+
+          # Add some more custom additional payload values here
+          options['tracing.auto_instrument.enabled'] = !defined?(Datadog::AutoInstrument::LOADED).nil?
+          options['tracing.writer_options.buffer_size'] =
+            format_configuration_value(configuration.tracing.writer_options[:buffer_size])
+          options['tracing.writer_options.flush_interval'] =
+            format_configuration_value(configuration.tracing.writer_options[:flush_interval])
+          options['logger.instance'] = configuration.logger.instance.class.to_s
+          options['appsec.enabled'] = configuration.dig('appsec', 'enabled') if configuration.respond_to?('appsec')
+          compact_hash(options)
+        end
+
+        def format_configuration_value(value)
+          # TODO: Add float if telemetry starts accepting it
+          case value
+          when Integer, String, true, false, nil
+            value
+          else
+            value.to_s
+          end
+        end
+
+        # Manual implementation of hash.compact used because it is unsupported by older Ruby versions (<2.4)
+        def compact_hash(hash)
+          hash.delete_if { |_k, v| v.nil? }
+        end
+
+        def env
+          Datadog.configuration.env
+        end
+
+        def service_name
+          Datadog.configuration.service
+        end
+
+        def service_version
+          Datadog.configuration.version
+        end
+
+        def library_version
+          Core::Environment::Identity.tracer_version
+        end
+
+        def products
+          Telemetry::V1::Product.new(profiler: profiler, appsec: appsec)
+        end
+
+        def profiler
+          { version: library_version }
+        end
+
+        def appsec
+          { version: library_version }
+        end
+
+        def agent_transport
+          adapter = Core::Configuration::AgentSettingsResolver.call(Datadog.configuration).adapter
+          if adapter == Datadog::Transport::Ext::UnixSocket::ADAPTER
+            'UDS'
+          else
+            'TCP'
+          end
+        end
+
+        def instrumented_integrations
+          Datadog.configuration.tracing.instrumented_integrations
+        end
+
+        def instrumented?(integration)
+          instrumented_integrations.include?(integration.name)
+        end
+
+        def patched?(integration)
+          !!integration.klass.patcher.patch_successful
+        end
+
+        def integration_auto_instrument?(integration)
+          integration.klass.auto_instrument?
+        end
+
+        def integration_compatible?(integration)
+          integration.klass.class.compatible?
+        end
+
+        def integration_version(integration)
+          integration.klass.class.version ? integration.klass.class.version.to_s : nil
+        end
+
+        def patch_error(integration)
+          patch_error_result = integration.klass.patcher.patch_error_result
+          if patch_error_result.nil? # if no error occurred during patching, but integration is still not instrumented
+            desc = "Available?: #{integration.klass.class.available?}"
+            desc += ", Loaded? #{integration.klass.class.loaded?}"
+            desc += ", Compatible? #{integration.klass.class.compatible?}"
+            desc += ", Patchable? #{integration.klass.class.patchable?}"
+            desc
+          else
+            patch_error_result.to_s
+          end
+        end
+      end
+      # rubocop:enable Metrics/ModuleLength
+    end
+  end
+end

data/lib/datadog/core/telemetry/emitter.rb

@@ -0,0 +1,48 @@
+# typed: true
+
+require_relative 'event'
+require_relative 'http/transport'
+require_relative '../utils/sequence'
+require_relative '../utils/forking'
+
+module Datadog
+  module Core
+    module Telemetry
+      # Class that emits telemetry events
+      class Emitter
+        attr_reader :http_transport
+
+        extend Core::Utils::Forking
+
+        # @param sequence [Datadog::Core::Utils::Sequence] Sequence object that stores and increments a counter
+        # @param http_transport [Datadog::Core::Telemetry::Http::Transport] Transport object that can be used to send
+        #   telemetry requests via the agent
+        def initialize(http_transport: Datadog::Core::Telemetry::Http::Transport.new)
+          @http_transport = http_transport
+        end
+
+        # Retrieves and emits a TelemetryRequest object based on the request type specified
+        # @param request_type [String] the type of telemetry request to collect data for
+        def request(request_type)
+          begin
+            request = Datadog::Core::Telemetry::Event.new.telemetry_request(
+              request_type: request_type,
+              seq_id: self.class.sequence.next
+            ).to_h
+            @http_transport.request(request_type: request_type.to_s, payload: request.to_json)
+          rescue StandardError => e
+            Datadog.logger.debug("Unable to send telemetry request for event `#{request_type}`: #{e}")
+            Telemetry::Http::InternalErrorResponse.new(e)
+          end
+        end
+
+        # Initializes a Sequence object to track seq_id if not already initialized; else returns stored
+        # Sequence object
+        def self.sequence
+          after_fork! { @sequence = Datadog::Core::Utils::Sequence.new(1) }
+          @sequence ||= Datadog::Core::Utils::Sequence.new(1)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/event.rb

@@ -0,0 +1,71 @@
+# typed: true
+
+require_relative 'collector'
+require_relative 'v1/app_event'
+require_relative 'v1/telemetry_request'
+
+module Datadog
+  module Core
+    module Telemetry
+      # Class defining methods to construct a Telemetry event
+      class Event
+        include Kernel
+
+        include Telemetry::Collector
+
+        API_VERSION = 'v1'.freeze
+
+        attr_reader \
+          :api_version
+
+        def initialize
+          @api_version = API_VERSION
+        end
+
+        # Forms a TelemetryRequest object based on the event request_type
+        # @param request_type [String] the type of telemetry request to collect data for
+        # @param seq_id [Integer] the ID of the request; incremented each time a telemetry request is sent to the API
+        def telemetry_request(request_type:, seq_id:)
+          Telemetry::V1::TelemetryRequest.new(
+            api_version: @api_version,
+            application: application,
+            host: host,
+            payload: payload(request_type),
+            request_type: request_type,
+            runtime_id: runtime_id,
+            seq_id: seq_id,
+            tracer_time: tracer_time,
+          )
+        end
+
+        private
+
+        def payload(request_type)
+          case request_type
+          when :'app-started'
+            app_started
+          when :'app-closing', :'app-heartbeat'
+            {}
+          when :'app-integrations-change'
+            app_integrations_change
+          else
+            raise ArgumentError, "Request type invalid, received request_type: #{@request_type}"
+          end
+        end
+
+        def app_started
+          Telemetry::V1::AppEvent.new(
+            dependencies: dependencies,
+            integrations: integrations,
+            configuration: configurations,
+            additional_payload: additional_payload
+          )
+        end
+
+        def app_integrations_change
+          Telemetry::V1::AppEvent.new(integrations: integrations)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/ext.rb

@@ -0,0 +1,11 @@
+# typed: true
+
+module Datadog
+  module Core
+    module Telemetry
+      module Ext
+        ENV_ENABLED = 'DD_INSTRUMENTATION_TELEMETRY_ENABLED'.freeze
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/heartbeat.rb

@@ -0,0 +1,37 @@
+# typed: false
+
+require_relative '../worker'
+require_relative '../workers/polling'
+
+module Datadog
+  module Core
+    module Telemetry
+      # Periodically (every DEFAULT_INTERVAL_SECONDS) sends a heartbeat event to the telemetry API.
+      class Heartbeat < Core::Worker
+        include Core::Workers::Polling
+
+        DEFAULT_INTERVAL_SECONDS = 60
+
+        def initialize(enabled: true, interval: DEFAULT_INTERVAL_SECONDS, &block)
+          # Workers::Polling settings
+          self.enabled = enabled
+          # Workers::IntervalLoop settings
+          self.loop_base_interval = interval
+          self.fork_policy = Core::Workers::Async::Thread::FORK_POLICY_STOP
+          super(&block)
+          start
+        end
+
+        def loop_wait_before_first_iteration?
+          true
+        end
+
+        private
+
+        def start
+          perform
+        end
+      end
+    end
+  end
+end