ddtrace 1.10.1 → 1.11.0

data/lib/datadog/core/configuration/option_set.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     module Configuration

data/lib/datadog/core/configuration/settings.rb

@@ -5,6 +5,7 @@ require_relative 'ext'
 require_relative '../environment/ext'
 require_relative '../runtime/ext'
 require_relative '../telemetry/ext'
+require_relative '../remote/ext'
 require_relative '../../profiling/ext'
 
 require_relative '../../tracing/configuration/settings'
@@ -202,9 +203,10 @@ module Datadog
           # @public_api
           settings :advanced do
             # This should never be reduced, as it can cause the resulting profiles to become biased.
-            # The current default should be enough for most services, allowing 16 threads to be sampled around 30 times
+            # The default should be enough for most services, allowing 16 threads to be sampled around 30 times
             # per second for a 60 second period.
-            # This setting is ignored when CPU Profiling 2.0 is in use.
+            #
+            # @deprecated This setting is ignored when CPU Profiling 2.0 is in use.
             option :max_events, default: 32768
 
             # Controls the maximum number of frames for each thread sampled. Can be tuned to avoid omitted frames in the
@@ -235,7 +237,7 @@ module Datadog
             # grouping and categorization of stack traces.
             option :code_provenance_enabled, default: true
 
-            # No longer does anything, and will be removed on dd-trace-rb 2.0.
+            # @deprecated No longer does anything, and will be removed on dd-trace-rb 2.0.
             #
             # This was added as a temporary support option in case of issues with the new `Profiling::HttpTransport` class
             # but we're now confident it's working nicely so we've removed the old code path.
@@ -251,29 +253,44 @@ module Datadog
             # Forces enabling the new CPU Profiling 2.0 profiler (see ddtrace release notes for more details).
             #
             # Note that setting this to "false" (or not setting it) will not prevent the new profiler from
-            # being automatically used in the future.
-            # This option will be deprecated for removal once the new profiler gets enabled by default for all customers.
+            # being automatically used.
+            # This option will be deprecated for removal once the legacy profiler is removed.
+            #
+            # @default `DD_PROFILING_FORCE_ENABLE_NEW` environment variable, otherwise `false`
             option :force_enable_new_profiler do |o|
               o.default { env_to_bool('DD_PROFILING_FORCE_ENABLE_NEW', false) }
               o.lazy
             end
 
+            # Forces enabling the *legacy* (non-CPU Profiling 2.0 profiler) even when it would otherwise NOT be enabled.
+            #
+            # Temporarily added to ease migration to the new CPU Profiling 2.0 profiler, and will be removed soon.
+            # Do not use unless instructed to by support.
+            # This option will be deprecated for removal once the legacy profiler is removed.
+            #
+            # @default `DD_PROFILING_FORCE_ENABLE_LEGACY` environment variable, otherwise `false`
+            option :force_enable_legacy_profiler do |o|
+              o.default { env_to_bool('DD_PROFILING_FORCE_ENABLE_LEGACY', false) }
+              o.lazy
+            end
+
             # Forces enabling of profiling of time/resources spent in Garbage Collection.
             #
             # Note that setting this to "false" (or not setting it) will not prevent the feature from being
             # being automatically enabled in the future.
             #
-            # This toggle was added because, although this feature is safe and enabled by default on Ruby 2.x,
-            # on Ruby 3.x it can break in applications that make use of Ractors due to two Ruby VM bugs:
-            # https://bugs.ruby-lang.org/issues/19112 AND https://bugs.ruby-lang.org/issues/18464.
-            #
-            # If you use Ruby 3.x and your application does not use Ractors (or if your Ruby has been patched), the
-            # feature is fully safe to enable and this toggle can be used to do so.
-            #
-            # Furthermore, currently this feature can add a lot of overhead for GC-heavy workloads.
+            # This feature defaults to off for two reasons:
+            # 1. Currently this feature can add a lot of overhead for GC-heavy workloads.
+            # 2. Although this feature is safe on Ruby 2.x, on Ruby 3.x it can break in applications that make use of
+            #    Ractors due to two Ruby VM bugs:
+            #    https://bugs.ruby-lang.org/issues/19112 AND https://bugs.ruby-lang.org/issues/18464.
+            #    If you use Ruby 3.x and your application does not use Ractors (or if your Ruby has been patched), the
+            #    feature is fully safe to enable and this toggle can be used to do so.
             #
             # We expect the once the above issues are overcome, we'll automatically enable the feature on fixed Ruby
             # versions.
+            #
+            # @default `DD_PROFILING_FORCE_ENABLE_GC` environment variable, otherwise `false`
             option :force_enable_gc_profiling do |o|
               o.default { env_to_bool('DD_PROFILING_FORCE_ENABLE_GC', false) }
               o.lazy
@@ -287,11 +304,26 @@ module Datadog
             #
             # If you use Ruby 3.x and your application does not use Ractors (or if your Ruby has been patched), the
             # feature is fully safe to enable and this toggle can be used to do so.
+            #
+            # @default `true` on Ruby 2.x, `false` on Ruby 3.x
             option :allocation_counting_enabled, default: RUBY_VERSION.start_with?('2.')
+
+            # Can be used to disable checking which version of `libmysqlclient` is being used by the `mysql2` gem.
+            #
+            # This setting is only used when the `mysql2` gem is installed.
+            #
+            # @default `DD_PROFILING_SKIP_MYSQL2_CHECK` environment variable, otherwise `false`
+            option :skip_mysql2_check do |o|
+              o.default { env_to_bool('DD_PROFILING_SKIP_MYSQL2_CHECK', false) }
+              o.lazy
+            end
           end
 
           # @public_api
           settings :upload do
+            # Network timeout for reporting profiling data to Datadog.
+            #
+            # @default `DD_PROFILING_UPLOAD_TIMEOUT` environment variable, otherwise `30.0`
             option :timeout_seconds do |o|
               o.setter { |value| value.nil? ? 30.0 : value.to_f }
               o.default { env_to_float(Profiling::Ext::ENV_UPLOAD_TIMEOUT, 30.0) }
@@ -436,11 +468,33 @@ module Datadog
         settings :telemetry do
           # Enable telemetry collection. This allows telemetry events to be emitted to the telemetry API.
           #
-          # @default `DD_INSTRUMENTATION_TELEMETRY_ENABLED` environment variable, otherwise `false`. In a future release,
-          #   this value will be changed to `true` by default as documented [here](https://docs.datadoghq.com/tracing/configure_data_security/#telemetry-collection).
+          # @default `DD_INSTRUMENTATION_TELEMETRY_ENABLED` environment variable, otherwise `true`.
+          #   Can be disabled as documented [here](https://docs.datadoghq.com/tracing/configure_data_security/#telemetry-collection).
+          # @return [Boolean]
+          option :enabled do |o|
+            o.default { env_to_bool(Core::Telemetry::Ext::ENV_ENABLED, true) }
+            o.lazy
+          end
+        end
+
+        # Remote configuration
+        # @public_api
+        settings :remote do
+          # Enable remote configuration. This allows fetching of remote configuration for live updates.
+          #
+          # @default `DD_REMOTE_CONFIGURATION_ENABLED` environment variable, otherwise `true`.
           # @return [Boolean]
           option :enabled do |o|
-            o.default { env_to_bool(Core::Telemetry::Ext::ENV_ENABLED, false) }
+            o.default { env_to_bool(Core::Remote::Ext::ENV_ENABLED, true) }
+            o.lazy
+          end
+
+          # Tune remote configuration polling interval.
+          #
+          # @default `DD_REMOTE_CONFIGURATION_POLL_INTERVAL_SECONDS` environment variable, otherwise `5.0` seconds.
+          # @return [Float]
+          option :poll_interval_seconds do |o|
+            o.default { env_to_float(Core::Remote::Ext::ENV_POLL_INTERVAL_SECONDS, 5.0) }
             o.lazy
           end
         end

data/lib/datadog/core/diagnostics/health.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../metrics/client'
 require_relative '../../tracing/diagnostics/health'
 

data/lib/datadog/core/environment/class_count.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     module Environment

data/lib/datadog/core/environment/gc.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     module Environment

data/lib/datadog/core/environment/identity.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'securerandom'
 
 require_relative 'ext'
@@ -47,9 +49,65 @@ module Datadog
           Core::Environment::Ext::LANG_VERSION
         end
 
+        # Returns tracer version, rubygems-style
         def tracer_version
           Core::Environment::Ext::TRACER_VERSION
         end
+
+        # Returns tracer version, comforming to https://semver.org/spec/v2.0.0.html
+        def tracer_version_semver2
+          # from ddtrace/version.rb, we have MAJOR.MINOR.PATCH plus optional .PRE and .BUILD
+          # - transform .PRE to -PRE if present
+          # - transform .BUILD to +BUILD if present
+          # - keep triplet segments before that
+
+          m = SEMVER2_RE.match(tracer_version)
+
+          pre = "-#{m[:pre]}" if m[:pre]
+          build = "+gha#{m[:gha_run_id]}.g#{m[:git_sha]}.#{m[:branch].tr('.', '-')}" if m[:build]
+
+          "#{m[:major]}.#{m[:minor]}.#{m[:patch]}#{pre}#{build}"
+        end
+
+        SEMVER2_RE = /
+          ^
+          # mandatory segments
+          (?<major>\d+)
+          \.
+          (?<minor>\d+)
+          \.
+          (?<patch>\d+)
+
+          # pre segments start with a value
+          # - containing at least one alpha
+          # - that is not part of our build segments expected values
+          # and stop with a value that is not part of our build segments expected values
+          (?:
+            \.
+            (?<pre>
+              (?!gha)
+              [a-zA-Z0-9]*[a-zA-Z][a-zA-Z0-9]*
+              (?:
+                \.
+                (?!gha)
+                [a-zA-Z0-9]+
+              )*
+            )
+          )?
+
+          # build segments: ours include CI info (`gha`), then git (`g`), then branch name
+          (?:
+            \.
+            (?<build>
+              gha(?<gha_run_id>\d+)
+              \.
+              g(?<git_sha>[a-f0-9]+)
+              \.
+              (?<branch>(?:[a-zA-Z0-9.])+)
+            )
+          )?
+          $
+        /xm.freeze
       end
     end
   end

data/lib/datadog/core/environment/socket.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'socket'
 require_relative '../utils/forking'
 

data/lib/datadog/core/environment/thread_count.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     module Environment

data/lib/datadog/core/environment/variable_helpers.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     # Namespace for handling application environment
@@ -14,11 +16,17 @@ module Datadog
         # @param [Boolean] default the default value if the keys in `var` are not present in the environment
         # @param [Boolean] deprecation_warning when `var` is a list, record a deprecation log when
         #   the first key in `var` is not used.
-        # @return [Boolean] if the environment value is the string `true`
+        # @return [Boolean] if the environment value is the string `true` or `1`
         # @return [default] if the environment value is not found
         def env_to_bool(var, default = nil, deprecation_warning: true)
           var = decode_array(var, deprecation_warning)
-          var && ENV.key?(var) ? ENV[var].to_s.strip.downcase == 'true' : default
+          if var && ENV.key?(var)
+            value = ENV[var].to_s.strip
+            value.downcase!
+            value == 'true' || value == '1' # rubocop:disable Style/MultipleComparison
+          else
+            default
+          end
         end
 
         # Reads an environment variable as an Integer.

data/lib/datadog/core/environment/vm_cache.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     module Environment

data/lib/datadog/core/extensions.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'configuration'
 
 # Global namespace that includes all Datadog functionality.

data/lib/datadog/core/header_collection.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     # A some-what abstract class representing a collection of headers.

data/lib/datadog/core/metrics/helpers.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     module Metrics

data/lib/datadog/core/metrics/metric.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     module Metrics

data/lib/datadog/core/remote/client/capabilities.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require_relative '../../../appsec/remote'
+
+module Datadog
+  module Core
+    module Remote
+      class Client
+        # Capbailities
+        class Capabilities
+          attr_reader :products, :capabilities, :receivers, :base64_capabilities
+
+          def initialize(settings)
+            @capabilities = []
+            @products = []
+            @receivers = []
+
+            register(settings)
+
+            @base64_capabilities = capabilities_to_base64
+          end
+
+          private
+
+          def register(settings)
+            if settings.appsec.enabled
+              register_capabilities(Datadog::AppSec::Remote.capabilities)
+              register_products(Datadog::AppSec::Remote.products)
+              register_receivers(Datadog::AppSec::Remote.receivers)
+            end
+          end
+
+          def register_capabilities(capabilities)
+            @capabilities.concat(capabilities)
+          end
+
+          def register_receivers(receivers)
+            @receivers.concat(receivers)
+          end
+
+          def register_products(products)
+            @products.concat(products)
+          end
+
+          def capabilities_to_base64
+            return '' if capabilities.empty?
+
+            cap_to_hexs = capabilities.reduce(:|).to_s(16).tap { |s| s.size.odd? && s.prepend('0') }.scan(/\h\h/)
+            binary = cap_to_hexs.each_with_object([]) { |hex, acc| acc << hex }.map { |e| e.to_i(16) }.pack('C*')
+
+            Base64.encode64(binary).chomp
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/remote/client.rb

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+require_relative 'configuration'
+require_relative 'dispatcher'
+
+module Datadog
+  module Core
+    module Remote
+      # Client communicates with the agent and sync remote configuration
+      class Client
+        class SyncError < StandardError; end
+
+        attr_reader :transport, :repository, :id, :dispatcher
+
+        def initialize(transport, capabilities, repository: Configuration::Repository.new)
+          @transport = transport
+
+          @repository = repository
+          @id = SecureRandom.uuid
+          @dispatcher = Dispatcher.new
+          @capabilities = capabilities
+
+          @capabilities.receivers.each do |receiver|
+            dispatcher.receivers << receiver
+          end
+        end
+
+        # rubocop:disable Metrics/AbcSize,Metrics/PerceivedComplexity,Metrics/MethodLength,Metrics/CyclomaticComplexity
+        def sync
+          # TODO: Skip sync if no capabilities are registered
+          response = transport.send_config(payload)
+
+          if response.ok?
+            # when response is completely empty, do nothing as in: leave as is
+            if response.empty?
+              Datadog.logger.debug { 'remote: empty response => NOOP' }
+
+              return
+            end
+
+            begin
+              paths = response.client_configs.map do |path|
+                Configuration::Path.parse(path)
+              end
+
+              targets = Configuration::TargetMap.parse(response.targets)
+
+              contents = Configuration::ContentList.parse(response.target_files)
+            rescue Remote::Configuration::Path::ParseError => e
+              raise SyncError, e.message
+            end
+
+            # To make sure steep does not complain
+            return unless paths && targets && contents
+
+            # TODO: sometimes it can strangely be so that paths.empty?
+            # TODO: sometimes it can strangely be so that targets.empty?
+
+            changes = repository.transaction do |current, transaction|
+              # paths to be removed: previously applied paths minus ingress paths
+              (current.paths - paths).each { |p| transaction.delete(p) }
+
+              # go through each ingress path
+              paths.each do |path|
+                # match target with path
+                target = targets[path]
+
+                # abort entirely if matching target not found
+                raise SyncError, "no target for path '#{path}'" if target.nil?
+
+                # new paths are not in previously applied paths
+                new = !current.paths.include?(path)
+
+                # updated paths are in previously applied paths
+                # but the content hash changed
+                changed = current.paths.include?(path) && !current.contents.find_content(path, target)
+
+                # skip if unchanged
+                same = !new && !changed
+
+                next if same
+
+                # match content with path and target
+                content = contents.find_content(path, target)
+
+                # abort entirely if matching content not found
+                raise SyncError, "no valid content for target at path '#{path}'" if content.nil?
+
+                # to be added or updated << config
+                # TODO: metadata (hash, version, etc...)
+                transaction.insert(path, target, content) if new
+                transaction.update(path, target, content) if changed
+              end
+
+              # save backend opaque backend state
+              transaction.set(opaque_backend_state: targets.opaque_backend_state)
+              transaction.set(targets_version: targets.version)
+
+              # upon transaction end, new list of applied config + metadata (add, change, remove) will be saved
+              # TODO: also remove stale config (matching removed) from cache (client configs is exhaustive list of paths)
+            end
+
+            if changes.empty?
+              Datadog.logger.debug { 'remote: no changes' }
+            else
+              dispatcher.dispatch(changes, repository)
+            end
+          end
+        end
+        # rubocop:enable Metrics/AbcSize,Metrics/PerceivedComplexity,Metrics/MethodLength,Metrics/CyclomaticComplexity
+
+        private
+
+        def payload # rubocop:disable Metrics/MethodLength
+          state = repository.state
+
+          client_tracer_tags = [
+            "platform:#{native_platform}", # native platform
+            # "asm.config.rules:#{}", # TODO: defined|undefined
+            # "asm.config.enabled:#{}", # TODO: true|false|undefined
+            "ruby.tracer.version:#{Core::Environment::Identity.tracer_version}",
+            "ruby.runtime.platform:#{RUBY_PLATFORM}",
+            "ruby.runtime.version:#{RUBY_VERSION}",
+            "ruby.runtime.engine.name:#{RUBY_ENGINE}",
+            "ruby.runtime.engine.version:#{ruby_engine_version}",
+            "ruby.rubygems.platform.local:#{Gem::Platform.local}",
+            "ruby.gem.libddwaf.version:#{gem_spec('libddwaf').version}",
+            "ruby.gem.libddwaf.platform:#{gem_spec('libddwaf').platform}",
+            "ruby.gem.libdatadog.version:#{gem_spec('libdatadog').version}",
+            "ruby.gem.libdatadog.platform:#{gem_spec('libdatadog').platform}",
+          ]
+
+          client_tracer = {
+            runtime_id: Core::Environment::Identity.id,
+            language: Core::Environment::Identity.lang,
+            tracer_version: tracer_version_semver2,
+            service: Datadog.configuration.service,
+            env: Datadog.configuration.env,
+            tags: client_tracer_tags,
+          }
+
+          app_version = Datadog.configuration.version
+
+          client_tracer[:app_version] = app_version if app_version
+
+          {
+            client: {
+              state: {
+                root_version: state.root_version,
+                targets_version: state.targets_version,
+                config_states: state.config_states,
+                has_error: state.has_error,
+                error: state.error,
+                backend_client_state: state.opaque_backend_state,
+              },
+              id: id,
+              products: @capabilities.products,
+              is_tracer: true,
+              is_agent: false,
+              client_tracer: client_tracer,
+              # base64 is needed otherwise the Go agent fails with an unmarshal error
+              capabilities: @capabilities.base64_capabilities
+            },
+            cached_target_files: state.cached_target_files,
+          }
+        end
+
+        def tracer_version_semver2
+          @tracer_version_semver2 ||= Core::Environment::Identity.tracer_version_semver2
+        end
+
+        def ruby_engine_version
+          @ruby_engine_version ||= defined?(RUBY_ENGINE_VERSION) ? RUBY_ENGINE_VERSION : RUBY_VERSION
+        end
+
+        def gem_spec(name)
+          (@gem_specs ||= {})[name] ||= ::Gem.loaded_specs[name] || GemSpecificationFallback.new(nil, nil)
+        end
+
+        def native_platform
+          return @native_platform unless @native_platform.nil?
+
+          os = if RUBY_ENGINE == 'jruby'
+                 os_name = java.lang.System.get_property('os.name')
+
+                 case os_name
+                 when /linux/i then 'linux'
+                 when /mac/i   then 'darwin'
+                 else os_name
+                 end
+               else
+                 Gem::Platform.local.os
+               end
+
+          version = if os != 'linux'
+                      nil
+                    elsif RUBY_PLATFORM =~ /linux-(.+)$/
+                      # Old rubygems don't handle non-gnu linux correctly
+                      Regexp.last_match(1)
+                    else
+                      'gnu'
+                    end
+
+          cpu = if RUBY_ENGINE == 'jruby'
+                  os_arch = java.lang.System.get_property('os.arch')
+
+                  case os_arch
+                  when 'amd64' then 'x86_64'
+                  when 'aarch64' then os == 'darwin' ? 'arm64' : 'aarch64'
+                  else os_arch
+                  end
+                else
+                  Gem::Platform.local.cpu
+                end
+
+          @native_platform = [cpu, os, version].compact.join('-')
+        end
+
+        GemSpecificationFallback = _ = Struct.new(:version, :platform) # rubocop:disable Naming/ConstantName
+      end
+    end
+  end
+end