datadog 2.0.0.beta1 → 2.0.0.beta2

data/lib/datadog/core/telemetry/event.rb

@@ -1,81 +1,271 @@
 # frozen_string_literal: true
 
-require_relative 'collector'
-require_relative 'v1/app_event'
-require_relative 'v1/telemetry_request'
-require_relative 'v2/app_client_configuration_change'
-
 module Datadog
   module Core
     module Telemetry
-      # Class defining methods to construct a Telemetry event
       class Event
-        include Telemetry::Collector
+        # Base class for all Telemetry V2 events.
+        class Base
+          # The type of the event.
+          # It must be one of the stings defined in the Telemetry V2
+          # specification for event names.
+          def type; end
+
+          # The JSON payload for the event.
+          # @param seq_id [Integer] The sequence ID for the event.
+          def payload(seq_id)
+            {}
+          end
+        end
+
+        # Telemetry class for the 'app-started' event
+        class AppStarted < Base
+          def type
+            'app-started'
+          end
+
+          def payload(seq_id)
+            @seq_id = seq_id
+            {
+              products: products,
+              configuration: configuration,
+              install_signature: install_signature,
+              # DEV: Not implemented yet
+              # error: error, # Start-up errors
+            }
+          end
+
+          private
+
+          def products
+            products = {
+              appsec: {
+                enabled: Datadog::AppSec.enabled?,
+              },
+              profiler: {
+                enabled: Datadog::Profiling.enabled?,
+              },
+              # DEV: Not implemented yet
+              # dynamic_instrumentation: {
+              #   enabled: true,
+              # }
+            }
+
+            if (unsupported_reason = Datadog::Profiling.unsupported_reason)
+              products[:profiler][:error] = {
+                code: 1, # Error code. 0 if no error.
+                message: unsupported_reason,
+              }
+            end
+
+            products
+          end
+
+          TARGET_OPTIONS = %w[
+            logger.level
+            profiling.advanced.code_provenance_enabled
+            profiling.advanced.endpoint.collection.enabled
+            profiling.enabled
+            runtime_metrics.enabled
+            tracing.analytics.enabled
+            tracing.propagation_style_extract
+            tracing.propagation_style_inject
+            tracing.enabled
+            tracing.log_injection
+            tracing.partial_flush.enabled
+            tracing.partial_flush.min_spans_threshold
+            tracing.report_hostname
+            tracing.sampling.rate_limit
+          ].freeze
+
+          # rubocop:disable Metrics/AbcSize
+          def configuration
+            config = Datadog.configuration
+
+            list = [
+              conf_value('DD_AGENT_HOST', config.agent.host),
+              conf_value('DD_AGENT_TRANSPORT', agent_transport(config)),
+              conf_value('DD_TRACE_SAMPLE_RATE', to_value(config.tracing.sampling.default_rate)),
+              conf_value(
+                'DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED',
+                config.tracing.contrib.global_default_service_name.enabled
+              ),
+            ]
 
-        API_VERSION = 'v1'
+            peer_service_mapping_str = ''
+            unless config.tracing.contrib.peer_service_mapping.empty?
+              peer_service_mapping = config.tracing.contrib.peer_service_mapping
+              peer_service_mapping_str = peer_service_mapping.map { |key, value| "#{key}:#{value}" }.join(',')
+            end
+            list << conf_value('DD_TRACE_PEER_SERVICE_MAPPING', peer_service_mapping_str)
 
-        attr_reader \
-          :api_version
+            # Whitelist of configuration options to send in additional payload object
+            TARGET_OPTIONS.each do |option|
+              split_option = option.split('.')
+              list << conf_value(option, to_value(config.dig(*split_option)))
+            end
+
+            # Add some more custom additional payload values here
+            list.push(
+              conf_value('tracing.auto_instrument.enabled', !defined?(Datadog::AutoInstrument::LOADED).nil?),
+              conf_value('tracing.writer_options.buffer_size', to_value(config.tracing.writer_options[:buffer_size])),
+              conf_value('tracing.writer_options.flush_interval', to_value(config.tracing.writer_options[:flush_interval])),
+              conf_value('tracing.opentelemetry.enabled', !defined?(Datadog::OpenTelemetry::LOADED).nil?),
+            )
+            list << conf_value('logger.instance', config.logger.instance.class.to_s) if config.logger.instance
+            list << conf_value('appsec.enabled', config.dig('appsec', 'enabled')) if config.respond_to?('appsec')
+            list << conf_value('ci.enabled', config.dig('ci', 'enabled')) if config.respond_to?('ci')
+
+            list.reject! { |entry| entry[:value].nil? }
+            list
+          end
+          # rubocop:enable Metrics/AbcSize
+
+          def agent_transport(config)
+            adapter = Core::Configuration::AgentSettingsResolver.call(config).adapter
+            if adapter == Datadog::Core::Transport::Ext::UnixSocket::ADAPTER
+              'UDS'
+            else
+              'TCP'
+            end
+          end
+
+          def conf_value(name, value, origin = 'code')
+            {
+              name: name,
+              value: value,
+              origin: origin,
+              seq_id: @seq_id,
+            }
+          end
+
+          def to_value(value)
+            # TODO: Add float if telemetry starts accepting it
+            case value
+            when Integer, String, true, false, nil
+              value
+            else
+              value.to_s
+            end
+          end
 
-        def initialize
-          @api_version = API_VERSION
+          def install_signature
+            config = Datadog.configuration
+            {
+              install_id: config.dig('telemetry', 'install_id'),
+              install_type: config.dig('telemetry', 'install_type'),
+              install_time: config.dig('telemetry', 'install_time'),
+            }
+          end
         end
 
-        # Forms a TelemetryRequest object based on the event request_type
-        # @param request_type [String] the type of telemetry request to collect data for
-        # @param seq_id [Integer] the ID of the request; incremented each time a telemetry request is sent to the API
-        # @param data [Object] arbitrary object to be passed to the respective `request_type` handler
-        def telemetry_request(request_type:, seq_id:, data: nil)
-          Telemetry::V1::TelemetryRequest.new(
-            api_version: @api_version,
-            application: application,
-            host: host,
-            payload: payload(request_type, data),
-            request_type: request_type,
-            runtime_id: runtime_id,
-            seq_id: seq_id,
-            tracer_time: tracer_time,
-          )
+        # Telemetry class for the 'app-dependencies-loaded' event
+        class AppDependenciesLoaded < Base
+          def type
+            'app-dependencies-loaded'
+          end
+
+          def payload(seq_id)
+            { dependencies: dependencies }
+          end
+
+          private
+
+          def dependencies
+            Gem.loaded_specs.collect do |name, gem|
+              {
+                name: name,
+                version: gem.version.to_s,
+                # hash: nil,
+              }
+            end
+          end
         end
 
-        private
+        # Telemetry class for the 'app-integrations-change' event
+        class AppIntegrationsChange < Base
+          def type
+            'app-integrations-change'
+          end
 
-        def payload(request_type, data)
-          case request_type
-          when :'app-started'
-            app_started
-          when :'app-closing', :'app-heartbeat'
-            {}
-          when :'app-integrations-change'
-            app_integrations_change
-          when 'app-client-configuration-change'
-            app_client_configuration_change(data)
-          else
-            raise ArgumentError, "Request type invalid, received request_type: #{@request_type}"
+          def payload(seq_id)
+            { integrations: integrations }
+          end
+
+          private
+
+          def integrations
+            instrumented_integrations = Datadog.configuration.tracing.instrumented_integrations
+            Datadog.registry.map do |integration|
+              is_instrumented = instrumented_integrations.include?(integration.name)
+
+              is_enabled = is_instrumented ? !!integration.klass.patcher.patch_successful : false
+
+              version = integration.klass.class.version ? integration.klass.class.version.to_s : nil
+
+              res = {
+                name: integration.name.to_s,
+                enabled: is_enabled,
+                version: version,
+                compatible: integration.klass.class.compatible?,
+                error: (patch_error(integration) if is_instrumented && !is_enabled),
+                # TODO: Track if integration is instrumented by manual configuration or by auto instrumentation
+                # auto_enabled: is_enabled && ???,
+              }
+              res.reject! { |_, v| v.nil? }
+              res
+            end
+          end
+
+          def patch_error(integration)
+            patch_error_result = integration.klass.patcher.patch_error_result
+            return patch_error_result.compact.to_s if patch_error_result
+
+            # If no error occurred during patching, but integration is still not instrumented
+            "Available?: #{integration.klass.class.available?}" \
+            ", Loaded? #{integration.klass.class.loaded?}" \
+            ", Compatible? #{integration.klass.class.compatible?}" \
+            ", Patchable? #{integration.klass.class.patchable?}"
           end
         end
 
-        def app_started
-          Telemetry::V1::AppEvent.new(
-            dependencies: dependencies,
-            integrations: integrations,
-            configuration: configurations,
-            additional_payload: additional_payload,
-            install_signature: install_signature
-          )
+        # Telemetry class for the 'app-client-configuration-change' event
+        class AppClientConfigurationChange < Base
+          def type
+            'app-client-configuration-change'
+          end
+
+          def initialize(changes, origin)
+            super()
+            @changes = changes
+            @origin = origin
+          end
+
+          def payload(seq_id)
+            {
+              configuration: @changes.map do |name, value|
+                {
+                  name: name,
+                  value: value,
+                  origin: @origin,
+                }
+              end
+            }
+          end
         end
 
-        def app_integrations_change
-          Telemetry::V1::AppEvent.new(integrations: integrations)
+        # Telemetry class for the 'app-heartbeat' event
+        class AppHeartbeat < Base
+          def type
+            'app-heartbeat'
+          end
         end
 
-        # DEV: During the transition from V1 to V2, the backend accepts many V2
-        # DEV: payloads through the V1 transport protocol.
-        # DEV: The `app-client-configuration-change` payload is one of them.
-        # DEV: Once V2 is fully implemented, `Telemetry::V2::AppClientConfigurationChange`
-        # DEV: should be reusable without major modifications.
-        def app_client_configuration_change(data)
-          Telemetry::V2::AppClientConfigurationChange.new(data[:changes], origin: data[:origin])
+        # Telemetry class for the 'app-closing' event
+        class AppClosing < Base
+          def type
+            'app-closing'
+          end
         end
       end
     end

data/lib/datadog/core/telemetry/ext.rb

@@ -6,6 +6,7 @@ module Datadog
       module Ext
         ENV_ENABLED = 'DD_INSTRUMENTATION_TELEMETRY_ENABLED'
         ENV_HEARTBEAT_INTERVAL = 'DD_TELEMETRY_HEARTBEAT_INTERVAL'
+        ENV_DEPENDENCY_COLLECTION = 'DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED'
         ENV_INSTALL_ID = 'DD_INSTRUMENTATION_INSTALL_ID'
         ENV_INSTALL_TYPE = 'DD_INSTRUMENTATION_INSTALL_TYPE'
         ENV_INSTALL_TIME = 'DD_INSTRUMENTATION_INSTALL_TIME'

data/lib/datadog/core/telemetry/heartbeat.rb

@@ -20,9 +20,7 @@ module Datadog
           start
         end
 
-        def loop_wait_before_first_iteration?
-          true
-        end
+        def loop_wait_before_first_iteration?; end
 
         private
 

data/lib/datadog/core/telemetry/http/ext.rb

@@ -10,9 +10,12 @@ module Datadog
           HEADER_CONTENT_LENGTH = 'Content-Length'
           HEADER_DD_TELEMETRY_API_VERSION = 'DD-Telemetry-API-Version'
           HEADER_DD_TELEMETRY_REQUEST_TYPE = 'DD-Telemetry-Request-Type'
+          HEADER_TELEMETRY_DEBUG_ENABLED = 'DD-Telemetry-Debug-Enabled'
+          HEADER_CLIENT_LIBRARY_LANGUAGE = 'DD-Client-Library-Language'
+          HEADER_CLIENT_LIBRARY_VERSION = 'DD-Client-Library-Version'
 
           CONTENT_TYPE_APPLICATION_JSON = 'application/json'
-          API_VERSION = 'v1'
+          API_VERSION = 'v2'
 
           AGENT_ENDPOINT = '/telemetry/proxy/api/v2/apmtelemetry'
         end

data/lib/datadog/core/telemetry/http/transport.rb

@@ -39,10 +39,15 @@ module Datadog
 
           def headers(request_type:, api_version: Http::Ext::API_VERSION)
             {
-              Datadog::Core::Transport::Ext::HTTP::HEADER_DD_INTERNAL_UNTRACED_REQUEST => '1',
-              Http::Ext::HEADER_CONTENT_TYPE => Http::Ext::CONTENT_TYPE_APPLICATION_JSON,
-              Http::Ext::HEADER_DD_TELEMETRY_API_VERSION => api_version,
-              Http::Ext::HEADER_DD_TELEMETRY_REQUEST_TYPE => request_type,
+              Core::Transport::Ext::HTTP::HEADER_DD_INTERNAL_UNTRACED_REQUEST => '1',
+              Ext::HEADER_CONTENT_TYPE => Http::Ext::CONTENT_TYPE_APPLICATION_JSON,
+              Ext::HEADER_DD_TELEMETRY_API_VERSION => api_version,
+              Ext::HEADER_DD_TELEMETRY_REQUEST_TYPE => request_type,
+              Ext::HEADER_CLIENT_LIBRARY_LANGUAGE => Core::Environment::Ext::LANG,
+              Ext::HEADER_CLIENT_LIBRARY_VERSION => Core::Environment::Identity.gem_datadog_version_semver2,
+
+              # Enable debug mode for telemetry
+              # HEADER_TELEMETRY_DEBUG_ENABLED => 'true',
             }
           end
 

data/lib/datadog/core/telemetry/request.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require_relative '../environment/platform'
+require_relative '../utils/hash'
+
+module Datadog
+  module Core
+    module Telemetry
+      # Module defining methods for collecting metadata for telemetry
+      module Request
+        class << self
+          using Core::Utils::Hash::Refinement
+
+          def build_payload(event, seq_id)
+            hash = {
+              api_version: Http::Ext::API_VERSION,
+              application: application,
+              debug: false,
+              host: host,
+              payload: event.payload(seq_id),
+              request_type: event.type,
+              runtime_id: Core::Environment::Identity.id,
+              seq_id: seq_id,
+              tracer_time: Time.now.to_i,
+            }
+            hash.compact!
+            hash
+          end
+
+          private
+
+          def application
+            config = Datadog.configuration
+            {
+              env: config.env,
+              language_name: Core::Environment::Ext::LANG,
+              language_version: Core::Environment::Ext::LANG_VERSION,
+              runtime_name: Core::Environment::Ext::RUBY_ENGINE,
+              runtime_version: Core::Environment::Ext::ENGINE_VERSION,
+              service_name: config.service,
+              service_version: config.version,
+              tracer_version: Core::Environment::Identity.gem_datadog_version_semver2
+            }
+          end
+
+          def host
+            {
+              architecture: Core::Environment::Platform.architecture,
+              hostname: Core::Environment::Platform.hostname,
+              kernel_name: Core::Environment::Platform.kernel_name,
+              kernel_release: Core::Environment::Platform.kernel_release,
+              kernel_version: Core::Environment::Platform.kernel_version
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/utils/base64.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Utils
+      # Helper methods for encoding and decoding base64
+      module Base64
+        def self.encode64(bin)
+          [bin].pack('m')
+        end
+
+        def self.strict_encode64(bin)
+          [bin].pack('m0')
+        end
+
+        def self.strict_decode64(str)
+          str.unpack1('m0')
+        end
+      end
+    end
+  end
+end

data/lib/datadog/opentelemetry/sdk/span_processor.rb

@@ -115,7 +115,7 @@ module Datadog
 
           # DEV: There's no `flat_map!`; we have to split it into two operations
           attributes = attributes.map do |key, value|
-            Datadog::OpenTelemetry::Trace::Span.serialize_attribute(key, value)
+            Datadog::Tracing::Utils.serialize_attribute(key, value)
           end
           attributes.flatten!(1)
 

data/lib/datadog/opentelemetry/sdk/trace/span.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'datadog/tracing/utils'
+
 module Datadog
   module OpenTelemetry
     module Trace
@@ -36,22 +38,6 @@ module Datadog
           span.set_error(status.description) if status && status.code == ::OpenTelemetry::Trace::Status::ERROR
         end
 
-        # Serialize values into Datadog span tags and metrics.
-        # Notably, arrays are exploded into many keys, each with
-        # a numeric suffix representing the array index, for example:
-        # `'foo' => ['a','b']` becomes `'foo.0' => 'a', 'foo.1' => 'b'`
-        def self.serialize_attribute(key, value)
-          if value.is_a?(Array)
-            value.flat_map.with_index do |v, idx|
-              serialize_attribute("#{key}.#{idx}", v)
-            end
-          elsif value.is_a?(TrueClass) || value.is_a?(FalseClass)
-            [[key, value.to_s]]
-          else
-            [[key, value]]
-          end
-        end
-
         # Create a meaningful Datadog operation name from the OpenTelemetry
         # semantic convention for span kind and span attributes.
         # @see https://opentelemetry.io/docs/specs/semconv/general/trace/
@@ -123,7 +109,7 @@ module Datadog
               span.name = rich_name.downcase
             end
 
-            Span.serialize_attribute(key, @attributes[key]).each do |new_key, value|
+            Tracing::Utils.serialize_attribute(key, @attributes[key]).each do |new_key, value|
               override_datadog_values(span, new_key, value)
 
               # When an attribute is used to override a Datadog Span property,

data/lib/datadog/profiling/collectors/code_provenance.rb

@@ -97,14 +97,20 @@ module Datadog
           end
         end
 
-        Library = Struct.new(:kind, :name, :version, :path) do
+        # Represents metadata we have for a ruby gem
+        class Library
+          attr_reader :kind, :name, :version, :path
+
           def initialize(kind:, name:, version:, path:)
-            super(kind.freeze, name.dup.freeze, version.to_s.dup.freeze, path.dup.freeze)
+            @kind = kind.freeze
+            @name = name.dup.freeze
+            @version = version.to_s.dup.freeze
+            @path = path.dup.freeze
             freeze
           end
 
-          def to_json(*args)
-            { kind: kind, name: name, version: version, paths: [path] }.to_json(*args)
+          def to_json(arg = nil)
+            { kind: @kind, name: @name, version: @version, paths: [@path] }.to_json(arg)
           end
         end
       end

data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb

@@ -24,6 +24,7 @@ module Datadog
           # **NOTE**: This should only be used for testing; disabling the dynamic sampling rate will increase the
           # profiler overhead!
           dynamic_sampling_rate_enabled: true,
+          skip_idle_samples_for_testing: false,
           idle_sampling_helper: IdleSamplingHelper.new
         )
           unless dynamic_sampling_rate_enabled
@@ -41,11 +42,14 @@ module Datadog
             dynamic_sampling_rate_enabled,
             dynamic_sampling_rate_overhead_target_percentage,
             allocation_profiling_enabled,
+            skip_idle_samples_for_testing,
           )
           @worker_thread = nil
           @failure_exception = nil
           @start_stop_mutex = Mutex.new
           @idle_sampling_helper = idle_sampling_helper
+          @wait_until_running_mutex = Mutex.new
+          @wait_until_running_condition = ConditionVariable.new
         end
 
         def start(on_failure_proc: nil)
@@ -108,6 +112,27 @@ module Datadog
           self.class._native_stats_reset_not_thread_safe(self)
           stats
         end
+
+        # Useful for testing, to e.g. make sure the profiler is running before we start running some code we want to observe
+        def wait_until_running(timeout_seconds: 5)
+          @wait_until_running_mutex.synchronize do
+            return true if self.class._native_is_running?(self)
+
+            @wait_until_running_condition.wait(@wait_until_running_mutex, timeout_seconds)
+
+            if self.class._native_is_running?(self)
+              true
+            else
+              raise "Timeout waiting for #{self.class.name} to start (waited for #{timeout_seconds} seconds)"
+            end
+          end
+        end
+
+        private
+
+        def signal_running
+          @wait_until_running_mutex.synchronize { @wait_until_running_condition.broadcast }
+        end
       end
     end
   end

data/lib/datadog/profiling/component.rb

@@ -111,19 +111,30 @@ module Datadog
       end
 
       private_class_method def self.enable_gc_profiling?(settings)
-        # See comments on the setting definition for more context on why it exists.
-        if settings.profiling.advanced.force_enable_gc_profiling
-          if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3')
-            Datadog.logger.debug(
-              'Profiling time/resources spent in Garbage Collection force enabled. Do not use Ractors in combination ' \
-              'with this option as profiles will be incomplete.'
-            )
-          end
+        return false unless settings.profiling.advanced.gc_enabled
 
-          true
-        else
-          false
+        # SEVERE - Only with Ractors
+        # On Ruby versions 3.0 (all), 3.1.0 to 3.1.3, and 3.2.0 to 3.2.2 gc profiling can trigger a VM bug
+        # that causes a segmentation fault during garbage collection of Ractors
+        # (https://bugs.ruby-lang.org/issues/18464). We don't allow enabling gc profiling on such Rubies.
+        # This bug is fixed on Ruby versions 3.1.4, 3.2.3 and 3.3.0.
+        if RUBY_VERSION.start_with?('3.0.') ||
+            (RUBY_VERSION.start_with?('3.1.') && RUBY_VERSION < '3.1.4') ||
+            (RUBY_VERSION.start_with?('3.2.') && RUBY_VERSION < '3.2.3')
+          Datadog.logger.warn(
+            "Current Ruby version (#{RUBY_VERSION}) has a VM bug where enabling GC profiling would cause "\
+            'crashes (https://bugs.ruby-lang.org/issues/18464). GC profiling has been disabled.'
+          )
+          return false
+        elsif RUBY_VERSION.start_with?('3.')
+          Datadog.logger.debug(
+            'In all known versions of Ruby 3.x, using Ractors may result in GC profiling unexpectedly ' \
+            'stopping (https://bugs.ruby-lang.org/issues/19112). Note that this stop has no impact in your ' \
+            'application stability or performance. This does not happen if Ractors are not used.'
+          )
         end
+
+        true
       end
 
       private_class_method def self.get_heap_sample_every(settings)
@@ -135,10 +146,7 @@ module Datadog
       end
 
       private_class_method def self.enable_allocation_profiling?(settings)
-        unless settings.profiling.allocation_enabled
-          # Allocation profiling disabled, short-circuit out
-          return false
-        end
+        return false unless settings.profiling.allocation_enabled
 
         # Allocation sampling is safe and supported on Ruby 2.x, but has a few caveats on Ruby 3.x.
 

data/lib/datadog/profiling/load_native_extension.rb

@@ -20,7 +20,20 @@ rescue LoadError => e
 end
 
 extension_name = "datadog_profiling_native_extension.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
-full_file_path = "#{__dir__}/../../#{extension_name}.#{RbConfig::CONFIG['DLEXT']}"
+file_name = "#{extension_name}.#{RbConfig::CONFIG['DLEXT']}"
+full_file_path = "#{__dir__}/../../#{file_name}"
+
+unless File.exist?(full_file_path)
+  extension_dir = Gem.loaded_specs['datadog'].extension_dir
+  candidate_path = "#{extension_dir}/#{file_name}"
+  if File.exist?(candidate_path)
+    full_file_path = candidate_path
+  else # rubocop:disable Style/EmptyElse
+    # We found none of the files. This is unexpected. Let's go ahead anyway, the error is going to be reported further
+    # down anyway.
+  end
+end
+
 init_function_name = "Init_#{extension_name.split('.').first}"
 
 status, result = Datadog::Profiling::Loader._native_load(full_file_path, init_function_name)