ddtrace 1.21.1 → 1.22.0

data/lib/datadog/core/configuration/settings.rb

@@ -266,7 +266,10 @@ module Datadog
 
             # Can be used to disable the gathering of names and versions of gems in use by the service, used to power
             # grouping and categorization of stack traces.
-            option :code_provenance_enabled, default: true
+            option :code_provenance_enabled do |o|
+              o.type :bool
+              o.default true
+            end
 
             # @deprecated No longer does anything, and will be removed on dd-trace-rb 2.0.
             #
@@ -313,27 +316,35 @@ module Datadog
               end
             end
 
-            # Forces enabling of profiling of time/resources spent in Garbage Collection.
+            # @deprecated No longer does anything, and will be removed on dd-trace-rb 2.0.
             #
-            # Note that setting this to "false" (or not setting it) will not prevent the feature from being
-            # being automatically enabled in the future.
+            # GC profiling is now on by default and controlled by {:gc_enabled}.
+            option :force_enable_gc_profiling do |o|
+              o.after_set do |_, _, precedence|
+                unless precedence == Datadog::Core::Configuration::Option::Precedence::DEFAULT
+                  Datadog.logger.warn(
+                    'The profiling.advanced.force_enable_gc_profiling setting has been deprecated for removal and no ' \
+                    'longer does anything (the feature is now on by default). ' \
+                    'Please remove this setting from your Datadog.configure block.'
+                  )
+                end
+              end
+            end
+
+            # Can be used to enable/disable garbage collection profiling.
             #
-            # This feature defaults to off for two reasons:
-            # 1. Currently this feature can add a lot of overhead for GC-heavy workloads.
-            # 2. Although this feature is safe on Ruby 2.x, on Ruby 3.x it can break in applications that make use of
-            #    Ractors due to two Ruby VM bugs:
-            #    https://bugs.ruby-lang.org/issues/19112 AND https://bugs.ruby-lang.org/issues/18464.
-            #    If you use Ruby 3.x and your application does not use Ractors (or if your Ruby has been patched), the
-            #    feature is fully safe to enable and this toggle can be used to do so.
+            # @warn To avoid https://bugs.ruby-lang.org/issues/18464 even when enabled, GC profiling is only started
+            #       for Ruby versions 2.x, 3.1.4+, 3.2.3+ and 3.3.0+
+            #       (more details in {Datadog::Profiling::Component.enable_gc_profiling?})
             #
-            # We expect the once the above issues are overcome, we'll automatically enable the feature on fixed Ruby
-            # versions.
+            # @warn Due to a VM bug in the Ractor implementation (https://bugs.ruby-lang.org/issues/19112) this feature
+            #       stops working when Ractors get garbage collected.
             #
-            # @default `DD_PROFILING_FORCE_ENABLE_GC` environment variable, otherwise `false`
-            option :force_enable_gc_profiling do |o|
-              o.env 'DD_PROFILING_FORCE_ENABLE_GC'
+            # @default `DD_PROFILING_GC_ENABLED` environment variable, otherwise `true`
+            option :gc_enabled do |o|
               o.type :bool
-              o.default false
+              o.env 'DD_PROFILING_GC_ENABLED'
+              o.default true
             end
 
             # Can be used to enable/disable the Datadog::Profiling.allocation_count feature.
@@ -687,6 +698,16 @@ module Datadog
         # Client-side telemetry configuration
         # @public_api
         settings :telemetry do
+          # Whether the bundled Ruby gems as reported through telemetry.
+          #
+          # @default `DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED` environment variable, otherwise `true`.
+          # @return [Boolean]
+          option :dependency_collection do |o|
+            o.type :bool
+            o.env Core::Telemetry::Ext::ENV_DEPENDENCY_COLLECTION
+            o.default true
+          end
+
           # Enable telemetry collection. This allows telemetry events to be emitted to the telemetry API.
           #
           # @default `DD_INSTRUMENTATION_TELEMETRY_ENABLED` environment variable, otherwise `true`.

data/lib/datadog/core/configuration.rb

@@ -81,18 +81,23 @@ module Datadog
         configuration = self.configuration
         yield(configuration)
 
-        safely_synchronize do |write_components|
+        built_components = false
+
+        components = safely_synchronize do |write_components|
           write_components.call(
             if components?
               replace_components!(configuration, @components)
             else
               components = build_components(configuration)
-              components.telemetry.started!
+              built_components = true
               components
             end
           )
         end
 
+        # Should only be called the first time components are built
+        components.telemetry.started! if built_components
+
         configuration
       end
 
@@ -192,9 +197,20 @@ module Datadog
         current_components = COMPONENTS_READ_LOCK.synchronize { defined?(@components) && @components }
         return current_components if current_components || !allow_initialization
 
-        safely_synchronize do |write_components|
-          (defined?(@components) && @components) || write_components.call(build_components(configuration))
+        built_components = false
+
+        components = safely_synchronize do |write_components|
+          if defined?(@components) && @components
+            @components
+          else
+            built_components = true
+            write_components.call(build_components(configuration))
+          end
         end
+
+        # Should only be called the first time components are built
+        components.telemetry.started! if built_components && components && components.telemetry
+        components
       end
 
       private

data/lib/datadog/core/environment/platform.rb

@@ -9,12 +9,18 @@ module Datadog
       module Platform
         module_function
 
+        # @return [String] ISA of host; `uname -m`
+        def architecture
+          Identity.lang_version >= '2.2' ? Etc.uname[:machine] : Gem::Platform.local.cpu
+        end
+
         # @return [String] name of host; `uname -n`
         def hostname
           Identity.lang_version >= '2.2' ? Etc.uname[:nodename] : nil
         end
 
-        # @return [String] name of kernel; `uname -s`
+        # System name, normally `Linux` or `Darwin` (but 'Mac OS X' on JRuby);
+        # @return [String] name of kernel; `uname -s`.
         def kernel_name
           Identity.lang_version >= '2.2' ? Etc.uname[:sysname] : Gem::Platform.local.os.capitalize
         end

data/lib/datadog/core/remote/client/capabilities.rb

@@ -53,7 +53,7 @@ module Datadog
             cap_to_hexs = capabilities.reduce(:|).to_s(16).tap { |s| s.size.odd? && s.prepend('0') }.scan(/\h\h/)
             binary = cap_to_hexs.each_with_object([]) { |hex, acc| acc << hex }.map { |e| e.to_i(16) }.pack('C*')
 
-            Base64.encode64(binary).chomp
+            Base64.strict_encode64(binary)
           end
         end
       end

data/lib/datadog/core/remote/transport/http/config.rb

@@ -51,7 +51,7 @@ module Datadog
 
                 # TODO: these fallbacks should be improved
                 roots = payload[:roots] || []
-                targets = payload[:targets] || Base64.encode64('{}').chomp
+                targets = payload[:targets] || Base64.strict_encode64('{}')
                 target_files = payload[:target_files] || []
                 client_configs = payload[:client_configs] || []
 

data/lib/datadog/core/telemetry/client.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'emitter'
+require_relative 'event'
 require_relative 'heartbeat'
 require_relative '../utils/forking'
 
@@ -10,21 +11,25 @@ module Datadog
       # Telemetry entrypoint, coordinates sending telemetry events at various points in app lifecycle.
       class Client
         attr_reader \
-          :emitter,
           :enabled,
-          :unsupported,
-          :worker
+          :unsupported
 
         include Core::Utils::Forking
 
         # @param enabled [Boolean] Determines whether telemetry events should be sent to the API
         # @param heartbeat_interval_seconds [Float] How frequently heartbeats will be reported, in seconds.
-        def initialize(heartbeat_interval_seconds:, enabled: true)
+        # @param [Boolean] dependency_collection Whether to send the `app-dependencies-loaded` event
+        def initialize(heartbeat_interval_seconds:, dependency_collection:, enabled: true)
           @enabled = enabled
           @emitter = Emitter.new
           @stopped = false
           @unsupported = false
+          @started = false
+          @dependency_collection = dependency_collection
+
           @worker = Telemetry::Heartbeat.new(enabled: @enabled, heartbeat_interval_seconds: heartbeat_interval_seconds) do
+            next unless @started # `started!` should be the first event, thus ensure that `heartbeat!` is not sent first.
+
             heartbeat!
           end
         end
@@ -37,21 +42,24 @@ module Datadog
         def started!
           return if !@enabled || forked?
 
-          res = @emitter.request(:'app-started')
+          res = @emitter.request(Event::AppStarted.new)
 
           if res.not_found? # Telemetry is only supported by agent versions 7.34 and up
             Datadog.logger.debug('Agent does not support telemetry; disabling future telemetry events.')
             disable!
             @unsupported = true # Prevent telemetry from getting re-enabled
+            return res
           end
 
-          res
+          @emitter.request(Event::AppDependenciesLoaded.new) if @dependency_collection
+
+          @started = true
         end
 
         def emit_closing!
           return if !@enabled || forked?
 
-          @emitter.request(:'app-closing')
+          @emitter.request(Event::AppClosing.new)
         end
 
         def stop!
@@ -64,14 +72,14 @@ module Datadog
         def integrations_change!
           return if !@enabled || forked?
 
-          @emitter.request(:'app-integrations-change')
+          @emitter.request(Event::AppIntegrationsChange.new)
         end
 
         # Report configuration changes caused by Remote Configuration.
         def client_configuration_change!(changes)
           return if !@enabled || forked?
 
-          @emitter.request('app-client-configuration-change', data: { changes: changes, origin: 'remote_config' })
+          @emitter.request(Event::AppClientConfigurationChange.new(changes, 'remote_config'))
         end
 
         private
@@ -79,7 +87,7 @@ module Datadog
         def heartbeat!
           return if !@enabled || forked?
 
-          @emitter.request(:'app-heartbeat')
+          @emitter.request(Event::AppHeartbeat.new)
         end
       end
     end

data/lib/datadog/core/telemetry/emitter.rb

@@ -1,4 +1,4 @@
-require_relative 'event'
+require_relative 'request'
 require_relative 'http/transport'
 require_relative '../utils/sequence'
 require_relative '../utils/forking'
@@ -12,7 +12,6 @@ module Datadog
 
         extend Core::Utils::Forking
 
-        # @param sequence [Datadog::Core::Utils::Sequence] Sequence object that stores and increments a counter
         # @param http_transport [Datadog::Core::Telemetry::Http::Transport] Transport object that can be used to send
         #   telemetry requests via the agent
         def initialize(http_transport: Datadog::Core::Telemetry::Http::Transport.new)
@@ -20,18 +19,15 @@ module Datadog
         end
 
         # Retrieves and emits a TelemetryRequest object based on the request type specified
-        # @param request_type [String] the type of telemetry request to collect data for
-        # @param data [Object] arbitrary object to be passed to the respective `request_type` handler
-        def request(request_type, data: nil)
+        def request(event)
           begin
-            request = Datadog::Core::Telemetry::Event.new.telemetry_request(
-              request_type: request_type,
-              seq_id: self.class.sequence.next,
-              data: data,
-            ).to_h
-            @http_transport.request(request_type: request_type.to_s, payload: request.to_json)
-          rescue StandardError => e
-            Datadog.logger.debug("Unable to send telemetry request for event `#{request_type}`: #{e}")
+            seq_id = self.class.sequence.next
+            payload = Request.build_payload(event, seq_id)
+            res = @http_transport.request(request_type: event.type, payload: payload.to_json)
+            Datadog.logger.debug { "Telemetry sent for event `#{event.type}` (status code: #{res.code})" }
+            res
+          rescue => e
+            Datadog.logger.debug("Unable to send telemetry request for event `#{event.type rescue 'unknown'}`: #{e}")
             Telemetry::Http::InternalErrorResponse.new(e)
           end
         end

data/lib/datadog/core/telemetry/event.rb

@@ -1,81 +1,271 @@
 # frozen_string_literal: true
 
-require_relative 'collector'
-require_relative 'v1/app_event'
-require_relative 'v1/telemetry_request'
-require_relative 'v2/app_client_configuration_change'
-
 module Datadog
   module Core
     module Telemetry
-      # Class defining methods to construct a Telemetry event
       class Event
-        include Telemetry::Collector
+        # Base class for all Telemetry V2 events.
+        class Base
+          # The type of the event.
+          # It must be one of the stings defined in the Telemetry V2
+          # specification for event names.
+          def type; end
+
+          # The JSON payload for the event.
+          # @param seq_id [Integer] The sequence ID for the event.
+          def payload(seq_id)
+            {}
+          end
+        end
+
+        # Telemetry class for the 'app-started' event
+        class AppStarted < Base
+          def type
+            'app-started'
+          end
+
+          def payload(seq_id)
+            @seq_id = seq_id
+            {
+              products: products,
+              configuration: configuration,
+              install_signature: install_signature,
+              # DEV: Not implemented yet
+              # error: error, # Start-up errors
+            }
+          end
+
+          private
+
+          def products
+            products = {
+              appsec: {
+                enabled: Datadog::AppSec.enabled?,
+              },
+              profiler: {
+                enabled: Datadog::Profiling.enabled?,
+              },
+              # DEV: Not implemented yet
+              # dynamic_instrumentation: {
+              #   enabled: true,
+              # }
+            }
+
+            if (unsupported_reason = Datadog::Profiling.unsupported_reason)
+              products[:profiler][:error] = {
+                code: 1, # Error code. 0 if no error.
+                message: unsupported_reason,
+              }
+            end
+
+            products
+          end
+
+          TARGET_OPTIONS = %w[
+            logger.level
+            profiling.advanced.code_provenance_enabled
+            profiling.advanced.endpoint.collection.enabled
+            profiling.enabled
+            runtime_metrics.enabled
+            tracing.analytics.enabled
+            tracing.distributed_tracing.propagation_inject_style
+            tracing.distributed_tracing.propagation_extract_style
+            tracing.enabled
+            tracing.log_injection
+            tracing.partial_flush.enabled
+            tracing.partial_flush.min_spans_threshold
+            tracing.report_hostname
+            tracing.sampling.rate_limit
+          ].freeze
+
+          # rubocop:disable Metrics/AbcSize
+          def configuration
+            config = Datadog.configuration
+
+            list = [
+              conf_value('DD_AGENT_HOST', config.agent.host),
+              conf_value('DD_AGENT_TRANSPORT', agent_transport(config)),
+              conf_value('DD_TRACE_SAMPLE_RATE', to_value(config.tracing.sampling.default_rate)),
+              conf_value(
+                'DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED',
+                config.tracing.contrib.global_default_service_name.enabled
+              ),
+            ]
 
-        API_VERSION = 'v1'
+            peer_service_mapping_str = ''
+            unless config.tracing.contrib.peer_service_mapping.empty?
+              peer_service_mapping = config.tracing.contrib.peer_service_mapping
+              peer_service_mapping_str = peer_service_mapping.map { |key, value| "#{key}:#{value}" }.join(',')
+            end
+            list << conf_value('DD_TRACE_PEER_SERVICE_MAPPING', peer_service_mapping_str)
 
-        attr_reader \
-          :api_version
+            # Whitelist of configuration options to send in additional payload object
+            TARGET_OPTIONS.each do |option|
+              split_option = option.split('.')
+              list << conf_value(option, to_value(config.dig(*split_option)))
+            end
+
+            # Add some more custom additional payload values here
+            list.push(
+              conf_value('tracing.auto_instrument.enabled', !defined?(Datadog::AutoInstrument::LOADED).nil?),
+              conf_value('tracing.writer_options.buffer_size', to_value(config.tracing.writer_options[:buffer_size])),
+              conf_value('tracing.writer_options.flush_interval', to_value(config.tracing.writer_options[:flush_interval])),
+              conf_value('tracing.opentelemetry.enabled', !defined?(Datadog::OpenTelemetry::LOADED).nil?),
+            )
+            list << conf_value('logger.instance', config.logger.instance.class.to_s) if config.logger.instance
+            list << conf_value('appsec.enabled', config.dig('appsec', 'enabled')) if config.respond_to?('appsec')
+            list << conf_value('ci.enabled', config.dig('ci', 'enabled')) if config.respond_to?('ci')
+
+            list.reject! { |entry| entry[:value].nil? }
+            list
+          end
+          # rubocop:enable Metrics/AbcSize
+
+          def agent_transport(config)
+            adapter = Core::Configuration::AgentSettingsResolver.call(config).adapter
+            if adapter == Datadog::Core::Transport::Ext::UnixSocket::ADAPTER
+              'UDS'
+            else
+              'TCP'
+            end
+          end
+
+          def conf_value(name, value, origin = 'code')
+            {
+              name: name,
+              value: value,
+              origin: origin,
+              seq_id: @seq_id,
+            }
+          end
+
+          def to_value(value)
+            # TODO: Add float if telemetry starts accepting it
+            case value
+            when Integer, String, true, false, nil
+              value
+            else
+              value.to_s
+            end
+          end
 
-        def initialize
-          @api_version = API_VERSION
+          def install_signature
+            config = Datadog.configuration
+            {
+              install_id: config.dig('telemetry', 'install_id'),
+              install_type: config.dig('telemetry', 'install_type'),
+              install_time: config.dig('telemetry', 'install_time'),
+            }
+          end
         end
 
-        # Forms a TelemetryRequest object based on the event request_type
-        # @param request_type [String] the type of telemetry request to collect data for
-        # @param seq_id [Integer] the ID of the request; incremented each time a telemetry request is sent to the API
-        # @param data [Object] arbitrary object to be passed to the respective `request_type` handler
-        def telemetry_request(request_type:, seq_id:, data: nil)
-          Telemetry::V1::TelemetryRequest.new(
-            api_version: @api_version,
-            application: application,
-            host: host,
-            payload: payload(request_type, data),
-            request_type: request_type,
-            runtime_id: runtime_id,
-            seq_id: seq_id,
-            tracer_time: tracer_time,
-          )
+        # Telemetry class for the 'app-dependencies-loaded' event
+        class AppDependenciesLoaded < Base
+          def type
+            'app-dependencies-loaded'
+          end
+
+          def payload(seq_id)
+            { dependencies: dependencies }
+          end
+
+          private
+
+          def dependencies
+            Gem.loaded_specs.collect do |name, gem|
+              {
+                name: name,
+                version: gem.version.to_s,
+                # hash: nil,
+              }
+            end
+          end
         end
 
-        private
+        # Telemetry class for the 'app-integrations-change' event
+        class AppIntegrationsChange < Base
+          def type
+            'app-integrations-change'
+          end
 
-        def payload(request_type, data)
-          case request_type
-          when :'app-started'
-            app_started
-          when :'app-closing', :'app-heartbeat'
-            {}
-          when :'app-integrations-change'
-            app_integrations_change
-          when 'app-client-configuration-change'
-            app_client_configuration_change(data)
-          else
-            raise ArgumentError, "Request type invalid, received request_type: #{@request_type}"
+          def payload(seq_id)
+            { integrations: integrations }
+          end
+
+          private
+
+          def integrations
+            instrumented_integrations = Datadog.configuration.tracing.instrumented_integrations
+            Datadog.registry.map do |integration|
+              is_instrumented = instrumented_integrations.include?(integration.name)
+
+              is_enabled = is_instrumented ? !!integration.klass.patcher.patch_successful : false
+
+              version = integration.klass.class.version ? integration.klass.class.version.to_s : nil
+
+              res = {
+                name: integration.name.to_s,
+                enabled: is_enabled,
+                version: version,
+                compatible: integration.klass.class.compatible?,
+                error: (patch_error(integration) if is_instrumented && !is_enabled),
+                # TODO: Track if integration is instrumented by manual configuration or by auto instrumentation
+                # auto_enabled: is_enabled && ???,
+              }
+              res.reject! { |_, v| v.nil? }
+              res
+            end
+          end
+
+          def patch_error(integration)
+            patch_error_result = integration.klass.patcher.patch_error_result
+            return patch_error_result.compact.to_s if patch_error_result
+
+            # If no error occurred during patching, but integration is still not instrumented
+            "Available?: #{integration.klass.class.available?}" \
+            ", Loaded? #{integration.klass.class.loaded?}" \
+            ", Compatible? #{integration.klass.class.compatible?}" \
+            ", Patchable? #{integration.klass.class.patchable?}"
           end
         end
 
-        def app_started
-          Telemetry::V1::AppEvent.new(
-            dependencies: dependencies,
-            integrations: integrations,
-            configuration: configurations,
-            additional_payload: additional_payload,
-            install_signature: install_signature
-          )
+        # Telemetry class for the 'app-client-configuration-change' event
+        class AppClientConfigurationChange < Base
+          def type
+            'app-client-configuration-change'
+          end
+
+          def initialize(changes, origin)
+            super()
+            @changes = changes
+            @origin = origin
+          end
+
+          def payload(seq_id)
+            {
+              configuration: @changes.map do |name, value|
+                {
+                  name: name,
+                  value: value,
+                  origin: @origin,
+                }
+              end
+            }
+          end
         end
 
-        def app_integrations_change
-          Telemetry::V1::AppEvent.new(integrations: integrations)
+        # Telemetry class for the 'app-heartbeat' event
+        class AppHeartbeat < Base
+          def type
+            'app-heartbeat'
+          end
         end
 
-        # DEV: During the transition from V1 to V2, the backend accepts many V2
-        # DEV: payloads through the V1 transport protocol.
-        # DEV: The `app-client-configuration-change` payload is one of them.
-        # DEV: Once V2 is fully implemented, `Telemetry::V2::AppClientConfigurationChange`
-        # DEV: should be reusable without major modifications.
-        def app_client_configuration_change(data)
-          Telemetry::V2::AppClientConfigurationChange.new(data[:changes], origin: data[:origin])
+        # Telemetry class for the 'app-closing' event
+        class AppClosing < Base
+          def type
+            'app-closing'
+          end
         end
       end
     end

data/lib/datadog/core/telemetry/ext.rb

@@ -6,6 +6,7 @@ module Datadog
       module Ext
         ENV_ENABLED = 'DD_INSTRUMENTATION_TELEMETRY_ENABLED'
         ENV_HEARTBEAT_INTERVAL = 'DD_TELEMETRY_HEARTBEAT_INTERVAL'
+        ENV_DEPENDENCY_COLLECTION = 'DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED'
         ENV_INSTALL_ID = 'DD_INSTRUMENTATION_INSTALL_ID'
         ENV_INSTALL_TYPE = 'DD_INSTRUMENTATION_INSTALL_TYPE'
         ENV_INSTALL_TIME = 'DD_INSTRUMENTATION_INSTALL_TIME'

data/lib/datadog/core/telemetry/heartbeat.rb

@@ -20,9 +20,7 @@ module Datadog
           start
         end
 
-        def loop_wait_before_first_iteration?
-          true
-        end
+        def loop_wait_before_first_iteration?; end
 
         private
 

data/lib/datadog/core/telemetry/http/ext.rb

@@ -10,9 +10,12 @@ module Datadog
           HEADER_CONTENT_LENGTH = 'Content-Length'
           HEADER_DD_TELEMETRY_API_VERSION = 'DD-Telemetry-API-Version'
           HEADER_DD_TELEMETRY_REQUEST_TYPE = 'DD-Telemetry-Request-Type'
+          HEADER_TELEMETRY_DEBUG_ENABLED = 'DD-Telemetry-Debug-Enabled'
+          HEADER_CLIENT_LIBRARY_LANGUAGE = 'DD-Client-Library-Language'
+          HEADER_CLIENT_LIBRARY_VERSION = 'DD-Client-Library-Version'
 
           CONTENT_TYPE_APPLICATION_JSON = 'application/json'
-          API_VERSION = 'v1'
+          API_VERSION = 'v2'
 
           AGENT_ENDPOINT = '/telemetry/proxy/api/v2/apmtelemetry'
         end