ddtrace 1.10.1 → 1.11.0.beta1

data/lib/datadog/appsec/remote.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require_relative '../core/remote/dispatcher'
+require_relative 'processor/rule_merger'
+require_relative 'processor/rule_loader'
+
+module Datadog
+  module AppSec
+    # Remote
+    module Remote
+      class ReadError < StandardError; end
+      class NoRulesError < StandardError; end
+
+      class << self
+        CAP_ASM_ACTIVATION                = 1 << 1 # Remote activation via ASM_FEATURES product
+        CAP_ASM_IP_BLOCKING               = 1 << 2 # accept IP blocking data from ASM_DATA product
+        CAP_ASM_DD_RULES                  = 1 << 3 # read ASM rules from ASM_DD product
+        CAP_ASM_EXCLUSIONS                = 1 << 4 # exclusion filters (passlist) via ASM product
+        CAP_ASM_REQUEST_BLOCKING          = 1 << 5 # can block on request info
+        CAP_ASM_RESPONSE_BLOCKING         = 1 << 6 # can block on response info
+        CAP_ASM_USER_BLOCKING             = 1 << 7 # accept user blocking data from ASM_DATA product
+        CAP_ASM_CUSTOM_RULES              = 1 << 8 # accept custom rules
+        CAP_ASM_CUSTOM_BLOCKING_RESPONSE  = 1 << 9 # supports custom http code or redirect sa blocking response
+
+        # TODO: we need to dynamically add CAP_ASM_ACTIVATION once we support it
+        ASM_CAPABILITIES = [
+          CAP_ASM_IP_BLOCKING,
+          CAP_ASM_USER_BLOCKING,
+          CAP_ASM_CUSTOM_RULES,
+          CAP_ASM_EXCLUSIONS,
+          CAP_ASM_REQUEST_BLOCKING,
+          CAP_ASM_RESPONSE_BLOCKING,
+          CAP_ASM_DD_RULES,
+        ].freeze
+
+        ASM_PRODUCTS = [
+          'ASM_DD',       # Datadog employee issued configuration
+          'ASM',          # customer issued configuration (rulesets, passlist...)
+          'ASM_FEATURES', # capabilities
+          'ASM_DATA',     # config files (IP addresses or users for blocking)
+        ].freeze
+
+        def capabilities
+          remote_features_enabled? ? ASM_CAPABILITIES : []
+        end
+
+        def products
+          remote_features_enabled? ? ASM_PRODUCTS : []
+        end
+
+        # rubocop:disable Metrics/MethodLength
+        def receivers
+          return [] unless remote_features_enabled?
+
+          matcher = Core::Remote::Dispatcher::Matcher::Product.new(ASM_PRODUCTS)
+          receiver = Core::Remote::Dispatcher::Receiver.new(matcher) do |repository, changes|
+            changes.each do |change|
+              Datadog.logger.debug { "remote config change: '#{change.path}'" }
+            end
+
+            rules = []
+            data = []
+            overrides = []
+            exclusions = []
+
+            asm_data_config_types = ['blocked_ips', 'blocked_users']
+            asm_overrides_config_types = ['blocking', 'disabled_rules']
+
+            repository.contents.each do |content|
+              case content.path.product
+              when 'ASM_DD'
+                rules << parse_content(content)
+              when 'ASM_DATA'
+                data << parse_content(content) if asm_data_config_types.include?(content.path.config_id)
+              when 'ASM'
+                overrides << parse_content(content) if asm_overrides_config_types.include?(content.path.config_id)
+                exclusions << parse_content(content) if content.path.config_id == 'exclusion_filters'
+              end
+            end
+
+            if rules.empty?
+              settings_rules = AppSec::Processor::RuleLoader.load_rules(ruleset: Datadog.configuration.appsec.ruleset)
+
+              raise NoRulesError, 'no default rules available' unless settings_rules
+
+              rules = [settings_rules]
+            end
+
+            ruleset = AppSec::Processor::RuleMerger.merge(
+              rules: rules,
+              data: data,
+              overrides: overrides,
+              exclusions: exclusions,
+            )
+
+            Datadog::AppSec.reconfigure(ruleset: ruleset)
+          end
+
+          [receiver]
+        end
+        # rubocop:enable Metrics/MethodLength
+
+        private
+
+        def remote_features_enabled?
+          Datadog::AppSec.send(:default_setting?, :ruleset)
+        end
+
+        def parse_content(content)
+          data = content.data.read
+
+          content.data.rewind
+
+          raise ReadError, 'EOF reached' if data.nil?
+
+          JSON.parse(data)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/utils/http.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module AppSec
     module Utils

data/lib/datadog/appsec/utils.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module AppSec
     # Utilities for AppSec

data/lib/datadog/appsec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'appsec/configuration'
 require_relative 'appsec/extensions'
 
@@ -17,6 +19,22 @@ module Datadog
         appsec_component.processor if appsec_component
       end
 
+      def reconfigure(ruleset:)
+        appsec_component = components.appsec
+
+        return unless appsec_component
+
+        appsec_component.reconfigure(ruleset: ruleset)
+      end
+
+      def reconfigure_lock(&block)
+        appsec_component = components.appsec
+
+        return unless appsec_component
+
+        appsec_component.reconfigure_lock(&block)
+      end
+
       private
 
       def components
@@ -24,10 +42,6 @@ module Datadog
       end
     end
 
-    def self.writer
-      @writer ||= Writer.new
-    end
-
     # Expose AppSec to global shared objects
     Extensions.activate!
   end

data/lib/datadog/ci/configuration/components.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../flush'
 
 module Datadog

data/lib/datadog/ci/configuration/settings.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../ext/settings'
 
 module Datadog

data/lib/datadog/ci/contrib/cucumber/configuration/settings.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../../../../tracing/contrib/configuration/settings'
 require_relative '../ext'
 

data/lib/datadog/ci/contrib/cucumber/formatter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../../test'
 require_relative '../../ext/app_types'
 require_relative '../../ext/environment'

data/lib/datadog/ci/contrib/cucumber/instrumentation.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'formatter'
 
 module Datadog

data/lib/datadog/ci/contrib/cucumber/patcher.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../../../tracing/contrib/patcher'
 require_relative 'instrumentation'
 

data/lib/datadog/ci/contrib/rspec/configuration/settings.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../../../../tracing/contrib/configuration/settings'
 require_relative '../ext'
 

data/lib/datadog/ci/contrib/rspec/patcher.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../../../tracing/contrib/patcher'
 require_relative 'example'
 

data/lib/datadog/ci/extensions.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../core/configuration/settings'
 require_relative '../core/configuration/components'
 

data/lib/datadog/ci/flush.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../tracing/metadata/ext'
 require_relative '../tracing/flush'
 

data/lib/datadog/ci.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'core'
 require_relative 'tracing'
 require_relative 'tracing/contrib'

data/lib/datadog/core/buffer/cruby.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'random'
 
 module Datadog

data/lib/datadog/core/buffer/random.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     module Buffer

data/lib/datadog/core/buffer/thread_safe.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'random'
 
 module Datadog

data/lib/datadog/core/chunker.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     # Chunks list of elements into batches

data/lib/datadog/core/configuration/agent_settings_resolver.rb

@@ -66,9 +66,9 @@ module Datadog
 
         def call
           # A transport_options proc configured for unix domain socket overrides most of the logic on this file
-          if transport_options.adapter == Transport::Ext::UnixSocket::ADAPTER
+          if transport_options.adapter == Datadog::Transport::Ext::UnixSocket::ADAPTER
             return AgentSettings.new(
-              adapter: Transport::Ext::UnixSocket::ADAPTER,
+              adapter: Datadog::Transport::Ext::UnixSocket::ADAPTER,
               ssl: false,
               hostname: nil,
               port: nil,
@@ -98,9 +98,9 @@ module Datadog
           # If no agent settings have been provided, we try to connect using a local unix socket.
           # We only do so if the socket is present when `ddtrace` runs.
           if should_use_uds_fallback?
-            Transport::Ext::UnixSocket::ADAPTER
+            Datadog::Transport::Ext::UnixSocket::ADAPTER
           else
-            Transport::Ext::HTTP::ADAPTER
+            Datadog::Transport::Ext::HTTP::ADAPTER
           end
         end
 
@@ -204,9 +204,9 @@ module Datadog
             if configured_hostname.nil? &&
                 configured_port.nil? &&
                 deprecated_for_removal_transport_configuration_proc.nil? &&
-                File.exist?(Transport::Ext::UnixSocket::DEFAULT_PATH)
+                File.exist?(Datadog::Transport::Ext::UnixSocket::DEFAULT_PATH)
 
-              Transport::Ext::UnixSocket::DEFAULT_PATH
+              Datadog::Transport::Ext::UnixSocket::DEFAULT_PATH
             end
         end
 

data/lib/datadog/core/configuration/components.rb

@@ -6,6 +6,7 @@ require_relative '../runtime/metrics'
 require_relative '../telemetry/client'
 require_relative '../workers/runtime_metrics'
 
+require_relative '../remote/component'
 require_relative '../../tracing/component'
 require_relative '../../profiling/component'
 require_relative '../../appsec/component'
@@ -17,7 +18,6 @@ module Datadog
       class Components
         class << self
           include Datadog::Tracing::Component
-          include Datadog::Profiling::Component
 
           def build_health_metrics(settings)
             settings = settings.diagnostics.health_metrics
@@ -60,6 +60,7 @@ module Datadog
         attr_reader \
           :health_metrics,
           :logger,
+          :remote,
           :profiler,
           :runtime_metrics,
           :telemetry,
@@ -67,27 +68,20 @@ module Datadog
           :appsec
 
         def initialize(settings)
-          # Logger
           @logger = self.class.build_logger(settings)
 
           agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
 
-          # Tracer
+          @remote = Remote::Component.build(settings, agent_settings)
           @tracer = self.class.build_tracer(settings, agent_settings)
-
-          # Profiler
-          @profiler = self.class.build_profiler(settings, agent_settings, @tracer)
-
-          # Runtime metrics
+          @profiler = Datadog::Profiling::Component.build_profiler_component(
+            settings: settings,
+            agent_settings: agent_settings,
+            optional_tracer: @tracer,
+          )
           @runtime_metrics = self.class.build_runtime_metrics_worker(settings)
-
-          # Health metrics
           @health_metrics = self.class.build_health_metrics(settings)
-
-          # Telemetry
           @telemetry = self.class.build_telemetry(settings)
-
-          # AppSec
           @appsec = Datadog::AppSec::Component.build_appsec_component(settings)
         end
 
@@ -111,6 +105,9 @@ module Datadog
         # If it has another instance to compare to, it will compare
         # and avoid tearing down parts still in use.
         def shutdown!(replacement = nil)
+          # Shutdown remote configuration
+          remote.shutdown! if remote
+
           # Decommission AppSec
           appsec.shutdown! if appsec
 

data/lib/datadog/core/configuration/dependency_resolver.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'tsort'
 
 module Datadog

data/lib/datadog/core/configuration/option.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     module Configuration

data/lib/datadog/core/configuration/option_definition.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'option'
 
 module Datadog

data/lib/datadog/core/configuration/option_definition_set.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'dependency_resolver'
 
 module Datadog

data/lib/datadog/core/configuration/option_set.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     module Configuration

data/lib/datadog/core/configuration/settings.rb

@@ -5,6 +5,7 @@ require_relative 'ext'
 require_relative '../environment/ext'
 require_relative '../runtime/ext'
 require_relative '../telemetry/ext'
+require_relative '../remote/ext'
 require_relative '../../profiling/ext'
 
 require_relative '../../tracing/configuration/settings'
@@ -202,9 +203,10 @@ module Datadog
           # @public_api
           settings :advanced do
             # This should never be reduced, as it can cause the resulting profiles to become biased.
-            # The current default should be enough for most services, allowing 16 threads to be sampled around 30 times
+            # The default should be enough for most services, allowing 16 threads to be sampled around 30 times
             # per second for a 60 second period.
-            # This setting is ignored when CPU Profiling 2.0 is in use.
+            #
+            # @deprecated This setting is ignored when CPU Profiling 2.0 is in use.
             option :max_events, default: 32768
 
             # Controls the maximum number of frames for each thread sampled. Can be tuned to avoid omitted frames in the
@@ -235,7 +237,7 @@ module Datadog
             # grouping and categorization of stack traces.
             option :code_provenance_enabled, default: true
 
-            # No longer does anything, and will be removed on dd-trace-rb 2.0.
+            # @deprecated No longer does anything, and will be removed on dd-trace-rb 2.0.
             #
             # This was added as a temporary support option in case of issues with the new `Profiling::HttpTransport` class
             # but we're now confident it's working nicely so we've removed the old code path.
@@ -251,29 +253,44 @@ module Datadog
             # Forces enabling the new CPU Profiling 2.0 profiler (see ddtrace release notes for more details).
             #
             # Note that setting this to "false" (or not setting it) will not prevent the new profiler from
-            # being automatically used in the future.
-            # This option will be deprecated for removal once the new profiler gets enabled by default for all customers.
+            # being automatically used.
+            # This option will be deprecated for removal once the legacy profiler is removed.
+            #
+            # @default `DD_PROFILING_FORCE_ENABLE_NEW` environment variable, otherwise `false`
             option :force_enable_new_profiler do |o|
               o.default { env_to_bool('DD_PROFILING_FORCE_ENABLE_NEW', false) }
               o.lazy
             end
 
+            # Forces enabling the *legacy* (non-CPU Profiling 2.0 profiler) even when it would otherwise NOT be enabled.
+            #
+            # Temporarily added to ease migration to the new CPU Profiling 2.0 profiler, and will be removed soon.
+            # Do not use unless instructed to by support.
+            # This option will be deprecated for removal once the legacy profiler is removed.
+            #
+            # @default `DD_PROFILING_FORCE_ENABLE_LEGACY` environment variable, otherwise `false`
+            option :force_enable_legacy_profiler do |o|
+              o.default { env_to_bool('DD_PROFILING_FORCE_ENABLE_LEGACY', false) }
+              o.lazy
+            end
+
             # Forces enabling of profiling of time/resources spent in Garbage Collection.
             #
             # Note that setting this to "false" (or not setting it) will not prevent the feature from being
             # being automatically enabled in the future.
             #
-            # This toggle was added because, although this feature is safe and enabled by default on Ruby 2.x,
-            # on Ruby 3.x it can break in applications that make use of Ractors due to two Ruby VM bugs:
-            # https://bugs.ruby-lang.org/issues/19112 AND https://bugs.ruby-lang.org/issues/18464.
-            #
-            # If you use Ruby 3.x and your application does not use Ractors (or if your Ruby has been patched), the
-            # feature is fully safe to enable and this toggle can be used to do so.
-            #
-            # Furthermore, currently this feature can add a lot of overhead for GC-heavy workloads.
+            # This feature defaults to off for two reasons:
+            # 1. Currently this feature can add a lot of overhead for GC-heavy workloads.
+            # 2. Although this feature is safe on Ruby 2.x, on Ruby 3.x it can break in applications that make use of
+            #    Ractors due to two Ruby VM bugs:
+            #    https://bugs.ruby-lang.org/issues/19112 AND https://bugs.ruby-lang.org/issues/18464.
+            #    If you use Ruby 3.x and your application does not use Ractors (or if your Ruby has been patched), the
+            #    feature is fully safe to enable and this toggle can be used to do so.
             #
             # We expect the once the above issues are overcome, we'll automatically enable the feature on fixed Ruby
             # versions.
+            #
+            # @default `DD_PROFILING_FORCE_ENABLE_GC` environment variable, otherwise `false`
             option :force_enable_gc_profiling do |o|
               o.default { env_to_bool('DD_PROFILING_FORCE_ENABLE_GC', false) }
               o.lazy
@@ -287,11 +304,26 @@ module Datadog
             #
             # If you use Ruby 3.x and your application does not use Ractors (or if your Ruby has been patched), the
             # feature is fully safe to enable and this toggle can be used to do so.
+            #
+            # @default `true` on Ruby 2.x, `false` on Ruby 3.x
             option :allocation_counting_enabled, default: RUBY_VERSION.start_with?('2.')
+
+            # Can be used to disable checking which version of `libmysqlclient` is being used by the `mysql2` gem.
+            #
+            # This setting is only used when the `mysql2` gem is installed.
+            #
+            # @default `DD_PROFILING_SKIP_MYSQL2_CHECK` environment variable, otherwise `false`
+            option :skip_mysql2_check do |o|
+              o.default { env_to_bool('DD_PROFILING_SKIP_MYSQL2_CHECK', false) }
+              o.lazy
+            end
           end
 
           # @public_api
           settings :upload do
+            # Network timeout for reporting profiling data to Datadog.
+            #
+            # @default `DD_PROFILING_UPLOAD_TIMEOUT` environment variable, otherwise `30.0`
             option :timeout_seconds do |o|
               o.setter { |value| value.nil? ? 30.0 : value.to_f }
               o.default { env_to_float(Profiling::Ext::ENV_UPLOAD_TIMEOUT, 30.0) }
@@ -445,6 +477,29 @@ module Datadog
           end
         end
 
+        # Remote configuration
+        # @public_api
+        settings :remote do
+          # Enable remote configuration. This allows fetching of remote configuration for live updates.
+          #
+          # @default `DD_REMOTE_CONFIGURATION_ENABLED` environment variable, otherwise `false`. In a future release,
+          #   this value will be changed to `true` by default.
+          # @return [Boolean]
+          option :enabled do |o|
+            o.default { env_to_bool(Core::Remote::Ext::ENV_ENABLED, false) }
+            o.lazy
+          end
+
+          # Tune remote configuration polling interval.
+          #
+          # @default `DD_REMOTE_CONFIGURATION_POLL_INTERVAL_SECONDS` environment variable, otherwise `5.0` seconds.
+          # @return [Float]
+          option :poll_interval_seconds do |o|
+            o.default { env_to_float(Core::Remote::Ext::ENV_POLL_INTERVAL_SECONDS, 5.0) }
+            o.lazy
+          end
+        end
+
         # TODO: Tracing should manage its own settings.
         #       Keep this extension here for now to keep things working.
         extend Datadog::Tracing::Configuration::Settings

data/lib/datadog/core/diagnostics/health.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../metrics/client'
 require_relative '../../tracing/diagnostics/health'
 

data/lib/datadog/core/environment/class_count.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     module Environment

data/lib/datadog/core/environment/gc.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     module Environment

data/lib/datadog/core/environment/identity.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'securerandom'
 
 require_relative 'ext'

data/lib/datadog/core/environment/socket.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'socket'
 require_relative '../utils/forking'
 

data/lib/datadog/core/environment/thread_count.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     module Environment

data/lib/datadog/core/environment/variable_helpers.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     # Namespace for handling application environment
@@ -14,11 +16,17 @@ module Datadog
         # @param [Boolean] default the default value if the keys in `var` are not present in the environment
         # @param [Boolean] deprecation_warning when `var` is a list, record a deprecation log when
         #   the first key in `var` is not used.
-        # @return [Boolean] if the environment value is the string `true`
+        # @return [Boolean] if the environment value is the string `true` or `1`
         # @return [default] if the environment value is not found
         def env_to_bool(var, default = nil, deprecation_warning: true)
           var = decode_array(var, deprecation_warning)
-          var && ENV.key?(var) ? ENV[var].to_s.strip.downcase == 'true' : default
+          if var && ENV.key?(var)
+            value = ENV[var].to_s.strip
+            value.downcase!
+            value == 'true' || value == '1' # rubocop:disable Style/MultipleComparison
+          else
+            default
+          end
         end
 
         # Reads an environment variable as an Integer.

data/lib/datadog/core/environment/vm_cache.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     module Environment

data/lib/datadog/core/extensions.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'configuration'
 
 # Global namespace that includes all Datadog functionality.

data/lib/datadog/core/header_collection.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     # A some-what abstract class representing a collection of headers.

data/lib/datadog/core/metrics/helpers.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     module Metrics