datadog 2.10.0 → 2.12.2

data/lib/datadog/appsec/instrumentation/gateway.rb

@@ -1,35 +1,29 @@
 # frozen_string_literal: true
 
+require_relative 'gateway/middleware'
+
 module Datadog
   module AppSec
     # Instrumentation for AppSec
     module Instrumentation
       # Instrumentation gateway implementation
       class Gateway
-        # Instrumentation gateway middleware
-        class Middleware
-          attr_reader :key, :block
-
-          def initialize(key, &block)
-            @key = key
-            @block = block
-          end
-
-          def call(stack, env)
-            @block.call(stack, env)
-          end
-        end
-
-        private_constant :Middleware
-
         def initialize
           @middlewares = Hash.new { |h, k| h[k] = [] }
+          @pushed_events = {}
         end
 
+        # NOTE: Be careful with pushed names because every pushed event name
+        #       is recorded in order to provide an ability to any subscriber
+        #       to check wether an arbitrary event had happened.
+        #
+        # WARNING: If we start pushing generated names we should consider
+        #          limiting the storage of pushed names.
         def push(name, env, &block)
-          block ||= -> {}
+          @pushed_events[name] = true
 
-          middlewares_for_name = middlewares[name]
+          block ||= -> {}
+          middlewares_for_name = @middlewares[name]
 
           return [block.call, nil] if middlewares_for_name.empty?
 
@@ -48,14 +42,15 @@ module Datadog
         end
 
         def watch(name, key, &block)
-          @middlewares[name] << Middleware.new(key, &block) unless middlewares[name].any? { |m| m.key == key }
+          @middlewares[name] << Middleware.new(key, &block) unless @middlewares[name].any? { |m| m.key == key }
         end
 
-        private
-
-        attr_reader :middlewares
+        def pushed?(name)
+          @pushed_events.key?(name)
+        end
       end
 
+      # NOTE: This left as-is and will be depricated soon.
       def self.gateway
         @gateway ||= Gateway.new # TODO: not thread safe
       end

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 require_relative '../../instrumentation/gateway'
-require_relative '../../reactive/engine'
-require_relative '../reactive/set_user'
 
 module Datadog
   module AppSec
@@ -19,31 +17,27 @@ module Datadog
 
             def watch_user_id(gateway = Instrumentation.gateway)
               gateway.watch('identity.set_user', :appsec) do |stack, user|
-                event = nil
                 context = Datadog::AppSec.active_context
-                engine = AppSec::Reactive::Engine.new
-
-                Monitor::Reactive::SetUser.subscribe(engine, context) do |result|
-                  if result.match?
-                    # TODO: should this hash be an Event instance instead?
-                    event = {
-                      waf_result: result,
-                      trace: context.trace,
-                      span: context.span,
-                      user: user,
-                      actions: result.actions
-                    }
-
-                    # We want to keep the trace in case of security event
-                    context.trace.keep! if context.trace
-                    Datadog::AppSec::Event.tag_and_keep!(context, result)
-                    context.events << event
-
-                    Datadog::AppSec::ActionsHandler.handle(result.actions)
-                  end
-                end
 
-                Monitor::Reactive::SetUser.publish(engine, user)
+                persistent_data = {
+                  'usr.id' => user.id
+                }
+
+                result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
+
+                if result.match?
+                  Datadog::AppSec::Event.tag_and_keep!(context, result)
+
+                  context.events << {
+                    waf_result: result,
+                    trace: context.trace,
+                    span: context.span,
+                    user: user,
+                    actions: result.actions
+                  }
+
+                  Datadog::AppSec::ActionsHandler.handle(result.actions)
+                end
 
                 stack.call(user)
               end

data/lib/datadog/appsec/processor/rule_merger.rb

@@ -22,7 +22,7 @@ module Datadog
           # TODO: `processors` and `scanners` are not provided by the caller, consider removing them
           def merge(
             telemetry:,
-            rules:, data: [], overrides: [], exclusions: [], custom_rules: [],
+            rules:, actions: [], data: [], overrides: [], exclusions: [], custom_rules: [],
             processors: nil, scanners: nil
           )
             processors ||= begin
@@ -54,6 +54,7 @@ module Datadog
             combined_exclusions = combine_exclusions(exclusions) if exclusions.any?
             combined_custom_rules = combine_custom_rules(custom_rules) if custom_rules.any?
 
+            combined_rules['actions'] = actions if actions.any?
             combined_rules['rules_data'] = combined_data if combined_data
             combined_rules['rules_override'] = combined_overrides if combined_overrides
             combined_rules['exclusions'] = combined_exclusions if combined_exclusions

data/lib/datadog/appsec/remote.rb

@@ -53,10 +53,12 @@ module Datadog
         end
 
         # rubocop:disable Metrics/MethodLength
+        # rubocop:disable Metrics/CyclomaticComplexity
         def receivers(telemetry)
           return [] unless remote_features_enabled?
 
           matcher = Core::Remote::Dispatcher::Matcher::Product.new(ASM_PRODUCTS)
+          # rubocop:disable Metrics/BlockLength
           receiver = Core::Remote::Dispatcher::Receiver.new(matcher) do |repository, changes|
             changes.each do |change|
               Datadog.logger.debug { "remote config change: '#{change.path}'" }
@@ -67,6 +69,7 @@ module Datadog
             data = []
             overrides = []
             exclusions = []
+            actions = []
 
             repository.contents.each do |content|
               parsed_content = parse_content(content)
@@ -80,6 +83,7 @@ module Datadog
                 overrides << parsed_content['rules_override'] if parsed_content['rules_override']
                 exclusions << parsed_content['exclusions'] if parsed_content['exclusions']
                 custom_rules << parsed_content['custom_rules'] if parsed_content['custom_rules']
+                actions.concat(parsed_content['actions']) if parsed_content['actions']
               end
             end
 
@@ -97,6 +101,7 @@ module Datadog
             ruleset = AppSec::Processor::RuleMerger.merge(
               rules: rules,
               data: data,
+              actions: actions,
               overrides: overrides,
               exclusions: exclusions,
               custom_rules: custom_rules,
@@ -104,11 +109,17 @@ module Datadog
             )
 
             Datadog::AppSec.reconfigure(ruleset: ruleset, telemetry: telemetry)
+
+            repository.contents.each do |content|
+              content.applied if ASM_PRODUCTS.include?(content.path.product)
+            end
           end
+          # rubocop:enable Metrics/BlockLength
 
           [receiver]
         end
         # rubocop:enable Metrics/MethodLength
+        # rubocop:enable Metrics/CyclomaticComplexity
 
         private
 

data/lib/datadog/appsec.rb

@@ -68,5 +68,8 @@ require_relative 'appsec/contrib/rails/integration'
 require_relative 'appsec/contrib/active_record/integration'
 require_relative 'appsec/contrib/devise/integration'
 require_relative 'appsec/contrib/graphql/integration'
+require_relative 'appsec/contrib/faraday/integration'
+require_relative 'appsec/contrib/excon/integration'
+require_relative 'appsec/contrib/rest_client/integration'
 
 require_relative 'appsec/autoload'

data/lib/datadog/core/configuration/components.rb

@@ -16,6 +16,8 @@ require_relative '../../appsec/component'
 require_relative '../../di/component'
 require_relative '../crashtracking/component'
 
+require_relative '../environment/agent_info'
+
 module Datadog
   module Core
     module Configuration
@@ -24,12 +26,12 @@ module Datadog
         class << self
           include Datadog::Tracing::Component
 
-          def build_health_metrics(settings)
+          def build_health_metrics(settings, logger)
             settings = settings.health_metrics
             options = { enabled: settings.enabled }
             options[:statsd] = settings.statsd unless settings.statsd.nil?
 
-            Core::Diagnostics::Health::Metrics.new(**options)
+            Core::Diagnostics::Health::Metrics.new(logger: logger, **options)
           end
 
           def build_logger(settings)
@@ -39,19 +41,20 @@ module Datadog
             logger
           end
 
-          def build_runtime_metrics(settings)
+          def build_runtime_metrics(settings, logger)
             options = { enabled: settings.runtime_metrics.enabled }
             options[:statsd] = settings.runtime_metrics.statsd unless settings.runtime_metrics.statsd.nil?
             options[:services] = [settings.service] unless settings.service.nil?
 
-            Core::Runtime::Metrics.new(**options)
+            Core::Runtime::Metrics.new(logger: logger, **options)
           end
 
-          def build_runtime_metrics_worker(settings)
+          def build_runtime_metrics_worker(settings, logger)
             # NOTE: Should we just ignore building the worker if its not enabled?
             options = settings.runtime_metrics.opts.merge(
               enabled: settings.runtime_metrics.enabled,
-              metrics: build_runtime_metrics(settings)
+              metrics: build_runtime_metrics(settings, logger),
+              logger: logger,
             )
 
             Core::Workers::RuntimeMetrics.new(options)
@@ -85,7 +88,8 @@ module Datadog
           :tracer,
           :crashtracker,
           :dynamic_instrumentation,
-          :appsec
+          :appsec,
+          :agent_info
 
         def initialize(settings)
           @logger = self.class.build_logger(settings)
@@ -96,9 +100,12 @@ module Datadog
           # the Core resolver from within your product/component's namespace.
           agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
 
+          # Exposes agent capability information for detection by any components
+          @agent_info = Core::Environment::AgentInfo.new(agent_settings)
+
           @telemetry = self.class.build_telemetry(settings, agent_settings, @logger)
 
-          @remote = Remote::Component.build(settings, agent_settings, telemetry: telemetry)
+          @remote = Remote::Component.build(settings, agent_settings, logger: @logger, telemetry: telemetry)
           @tracer = self.class.build_tracer(settings, agent_settings, logger: @logger)
           @crashtracker = self.class.build_crashtracker(settings, agent_settings, logger: @logger)
 
@@ -110,8 +117,8 @@ module Datadog
           )
           @environment_logger_extra.merge!(profiler_logger_extra) if profiler_logger_extra
 
-          @runtime_metrics = self.class.build_runtime_metrics_worker(settings)
-          @health_metrics = self.class.build_health_metrics(settings)
+          @runtime_metrics = self.class.build_runtime_metrics_worker(settings, @logger)
+          @health_metrics = self.class.build_health_metrics(settings, @logger)
           @appsec = Datadog::AppSec::Component.build_appsec_component(settings, telemetry: telemetry)
           @dynamic_instrumentation = Datadog::DI::Component.build(settings, agent_settings, @logger, telemetry: telemetry)
           @environment_logger_extra[:dynamic_instrumentation_enabled] = !!@dynamic_instrumentation

data/lib/datadog/core/configuration/ext.rb

@@ -37,7 +37,7 @@ module Datadog
           module UnixSocket
             ADAPTER = :unix
             DEFAULT_PATH = '/var/run/datadog/apm.socket'
-            DEFAULT_TIMEOUT_SECONDS = 1
+            DEFAULT_TIMEOUT_SECONDS = 30
           end
         end
       end

data/lib/datadog/core/configuration/option_definition.rb

@@ -79,6 +79,8 @@ module Datadog
             @deprecated_env = value
           end
 
+          # Invoked when the option is first read, and {#env} is defined.
+          # The block provided is only invoked if the environment variable is present (not-nil).
           def env_parser(&block)
             @env_parser = block
           end

data/lib/datadog/core/configuration/settings.rb

@@ -461,15 +461,31 @@ module Datadog
               end
             end
 
-            # Enables GVL profiling. This will show when threads are waiting for GVL in the timeline view.
-            #
-            # This is a preview feature and disabled by default. It requires Ruby 3.2+.
-            #
-            # @default `DD_PROFILING_PREVIEW_GVL_ENABLED` environment variable as a boolean, otherwise `false`
+            # @deprecated Use {:gvl_enabled} instead.
             option :preview_gvl_enabled do |o|
               o.type :bool
-              o.env 'DD_PROFILING_PREVIEW_GVL_ENABLED'
               o.default false
+              o.after_set do |_, _, precedence|
+                unless precedence == Datadog::Core::Configuration::Option::Precedence::DEFAULT
+                  Datadog.logger.warn(
+                    'The profiling.advanced.preview_gvl_enabled setting has been deprecated for removal and ' \
+                    'no longer does anything. Please remove it from your Datadog.configure block. ' \
+                    'GVL profiling is now controlled by the profiling.advanced.gvl_enabled setting instead.'
+                  )
+                end
+              end
+            end
+
+            # Controls GVL profiling. This will show when threads are waiting for GVL in the timeline view.
+            #
+            # This feature requires Ruby 3.2+.
+            #
+            # @default `DD_PROFILING_GVL_ENABLED` environment variable as a boolean, otherwise `true`
+            option :gvl_enabled do |o|
+              o.type :bool
+              o.deprecated_env 'DD_PROFILING_PREVIEW_GVL_ENABLED'
+              o.env 'DD_PROFILING_GVL_ENABLED'
+              o.default true
             end
 
             # Controls the smallest time period the profiler will report a thread waiting for the GVL.

data/lib/datadog/core/encoding.rb

@@ -10,6 +10,7 @@ module Datadog
       # Encoder interface that provides the logic to encode traces and service
       # @abstract
       module Encoder
+        # :nocov:
         def content_type
           raise NotImplementedError
         end
@@ -23,6 +24,13 @@ module Datadog
         def encode(_)
           raise NotImplementedError
         end
+
+        # Deserializes a value serialized with {#encode}.
+        # This method is used for debugging purposes.
+        def decode(_)
+          raise NotImplementedError
+        end
+        # :nocov:
       end
 
       # Encoder for the JSON format
@@ -41,6 +49,10 @@ module Datadog
           JSON.dump(obj)
         end
 
+        def decode(obj)
+          JSON.parse(obj)
+        end
+
         def join(encoded_data)
           "[#{encoded_data.join(',')}]"
         end
@@ -62,6 +74,10 @@ module Datadog
           MessagePack.pack(obj)
         end
 
+        def decode(obj)
+          MessagePack.unpack(obj)
+        end
+
         def join(encoded_data)
           packer = MessagePack::Packer.new
           packer.write_array_header(encoded_data.size)

data/lib/datadog/core/environment/agent_info.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Environment
+      # Retrieves the agent's `/info` endpoint data.
+      # This data can be used to determine the capabilities of the local Datadog agent.
+      #
+      # @example Example response payload
+      #   {
+      #     "version" : "7.57.2",
+      #     "git_commit" : "38ba0c7",
+      #     "endpoints" : [ "/v0.4/traces", "/v0.4/services", "/v0.7/traces", "/v0.7/config" ],
+      #     "client_drop_p0s" : true,
+      #     "span_meta_structs" : true,
+      #     "long_running_spans" : true,
+      #     "evp_proxy_allowed_headers" : [ "Content-Type", "Accept-Encoding", "Content-Encoding", "User-Agent" ],
+      #     "config" : {
+      #       "default_env" : "none",
+      #       "target_tps" : 10,
+      #       "max_eps" : 200,
+      #       "receiver_port" : 8126,
+      #       "receiver_socket" : "/var/run/datadog/apm.socket",
+      #       "connection_limit" : 0,
+      #       "receiver_timeout" : 0,
+      #       "max_request_bytes" : 26214400,
+      #       "statsd_port" : 8125,
+      #       "analyzed_spans_by_service" : { },
+      #       "obfuscation" : {
+      #         "elastic_search" : true,
+      #         "mongo" : true,
+      #         "sql_exec_plan" : false,
+      #         "sql_exec_plan_normalize" : false,
+      #         "http" : {
+      #           "remove_query_string" : false,
+      #           "remove_path_digits" : false
+      #         },
+      #         "remove_stack_traces" : false,
+      #         "redis" : {
+      #           "Enabled" : true,
+      #           "RemoveAllArgs" : false
+      #         },
+      #         "memcached" : {
+      #           "Enabled" : true,
+      #           "KeepCommand" : false
+      #         }
+      #       }
+      #     },
+      #     "peer_tags" : null
+      #   }
+      #
+      # @see https://github.com/DataDog/datadog-agent/blob/f07df0a3c1fca0c83b5a15f553bd994091b0c8ac/pkg/trace/api/info.go#L20
+      class AgentInfo
+        attr_reader :agent_settings
+
+        def initialize(agent_settings)
+          @agent_settings = agent_settings
+          @client = Remote::Transport::HTTP.root(agent_settings: agent_settings)
+        end
+
+        # Fetches the information from the agent.
+        # @return [Datadog::Core::Remote::Transport::HTTP::Negotiation::Response] the response from the agent
+        # @return [nil] if an error occurred while fetching the information
+        def fetch
+          res = @client.send_info
+          return unless res.ok?
+
+          res
+        end
+
+        def ==(other)
+          other.is_a?(self.class) && other.agent_settings == agent_settings
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/metrics/client.rb

@@ -21,9 +21,10 @@ module Datadog
         extend Options
         extend Helpers
 
-        attr_reader :statsd
+        attr_reader :statsd, :logger
 
-        def initialize(statsd: nil, enabled: true, **_)
+        def initialize(logger:, statsd: nil, enabled: true, **_)
+          @logger = logger
           @statsd =
             if supported?
               statsd || default_statsd_client
@@ -98,7 +99,7 @@ module Datadog
 
           statsd.count(stat, value, metric_options(options))
         rescue StandardError => e
-          Datadog.logger.error(
+          logger.error(
             "Failed to send count stat. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
           )
           Telemetry::Logger.report(e, description: 'Failed to send count stat')
@@ -112,7 +113,7 @@ module Datadog
 
           statsd.distribution(stat, value, metric_options(options))
         rescue StandardError => e
-          Datadog.logger.error(
+          logger.error(
             "Failed to send distribution stat. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
           )
           Telemetry::Logger.report(e, description: 'Failed to send distribution stat')
@@ -125,7 +126,7 @@ module Datadog
 
           statsd.increment(stat, metric_options(options))
         rescue StandardError => e
-          Datadog.logger.error(
+          logger.error(
             "Failed to send increment stat. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
           )
           Telemetry::Logger.report(e, description: 'Failed to send increment stat')
@@ -139,7 +140,7 @@ module Datadog
 
           statsd.gauge(stat, value, metric_options(options))
         rescue StandardError => e
-          Datadog.logger.error(
+          logger.error(
             "Failed to send gauge stat. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
           )
           Telemetry::Logger.report(e, description: 'Failed to send gauge stat')
@@ -159,7 +160,7 @@ module Datadog
             end
           rescue StandardError => e
             # TODO: Likely to be redundant, since `distribution` handles its own errors.
-            Datadog.logger.error(
+            logger.error(
               "Failed to send time stat. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
             )
             Telemetry::Logger.report(e, description: 'Failed to send time stat')
@@ -194,7 +195,7 @@ module Datadog
 
         def ignored_statsd_warning
           IGNORED_STATSD_ONLY_ONCE.run do
-            Datadog.logger.warn(
+            logger.warn(
               'Ignoring user-supplied statsd instance as currently-installed version of dogstastd-ruby is incompatible. ' \
               "To fix this, ensure that you have `gem 'dogstatsd-ruby', '~> 5.3'` on your Gemfile or gems.rb file."
             )

data/lib/datadog/core/remote/client.rb

@@ -13,10 +13,11 @@ module Datadog
         class TransportError < StandardError; end
         class SyncError < StandardError; end
 
-        attr_reader :transport, :repository, :id, :dispatcher
+        attr_reader :transport, :repository, :id, :dispatcher, :logger
 
-        def initialize(transport, capabilities, repository: Configuration::Repository.new)
+        def initialize(transport, capabilities, logger:, repository: Configuration::Repository.new)
           @transport = transport
+          @logger = logger
 
           @repository = repository
           @id = SecureRandom.uuid
@@ -40,7 +41,7 @@ module Datadog
         def process_response(response)
           # when response is completely empty, do nothing as in: leave as is
           if response.empty?
-            Datadog.logger.debug { 'remote: empty response => NOOP' }
+            logger.debug { 'remote: empty response => NOOP' }
 
             return
           end
@@ -112,7 +113,7 @@ module Datadog
           end
 
           if changes.empty?
-            Datadog.logger.debug { 'remote: no changes' }
+            logger.debug { 'remote: no changes' }
           else
             dispatcher.dispatch(changes, repository)
           end

data/lib/datadog/core/remote/component.rb

@@ -13,22 +13,24 @@ module Datadog
       # Configures the HTTP transport to communicate with the agent
       # to fetch and sync the remote configuration
       class Component
-        attr_reader :client, :healthy
+        attr_reader :logger, :client, :healthy
+
+        def initialize(settings, capabilities, agent_settings, logger:)
+          @logger = logger
 
-        def initialize(settings, capabilities, agent_settings)
           transport_options = {}
           transport_options[:agent_settings] = agent_settings if agent_settings
 
           negotiation = Negotiation.new(settings, agent_settings)
-          transport_v7 = Datadog::Core::Remote::Transport::HTTP.v7(**transport_options.dup)
+          transport_v7 = Datadog::Core::Remote::Transport::HTTP.v7(**transport_options) # steep:ignore
 
           @barrier = Barrier.new(settings.remote.boot_timeout_seconds)
 
-          @client = Client.new(transport_v7, capabilities)
+          @client = Client.new(transport_v7, capabilities, logger: logger)
           @healthy = false
-          Datadog.logger.debug { "new remote configuration client: #{@client.id}" }
+          logger.debug { "new remote configuration client: #{@client.id}" }
 
-          @worker = Worker.new(interval: settings.remote.poll_interval_seconds) do
+          @worker = Worker.new(interval: settings.remote.poll_interval_seconds, logger: logger) do
             unless @healthy || negotiation.endpoint?('/v0.7/config')
               @barrier.lift
 
@@ -40,7 +42,7 @@ module Datadog
               @healthy ||= true
             rescue Client::SyncError => e
               # Transient errors due to network or agent. Logged the error but not via telemetry
-              Datadog.logger.error do
+              logger.error do
                 "remote worker client sync error: #{e.message} location: #{Array(e.backtrace).first}. skipping sync"
               end
             rescue StandardError => e
@@ -50,15 +52,15 @@ module Datadog
               negotiation = Negotiation.new(settings, agent_settings)
 
               # Transient errors due to network or agent. Logged the error but not via telemetry
-              Datadog.logger.error do
+              logger.error do
                 "remote worker error: #{e.class.name} #{e.message} location: #{Array(e.backtrace).first}. "\
                 'reseting client state'
               end
 
               # client state is unknown, state might be corrupted
-              @client = Client.new(transport_v7, capabilities)
+              @client = Client.new(transport_v7, capabilities, logger: logger)
               @healthy = false
-              Datadog.logger.debug { "new remote configuration client: #{@client.id}" }
+              logger.debug { "new remote configuration client: #{@client.id}" }
 
               # TODO: bail out if too many errors?
             end
@@ -152,10 +154,10 @@ module Datadog
           #
           # Those checks are instead performed inside the worker loop.
           # This allows users to upgrade their agent while keeping their application running.
-          def build(settings, agent_settings, telemetry:)
+          def build(settings, agent_settings, logger:, telemetry:)
             return unless settings.remote.enabled
 
-            new(settings, Client::Capabilities.new(settings, telemetry), agent_settings)
+            new(settings, Client::Capabilities.new(settings, telemetry), agent_settings, logger: logger)
           end
         end
       end

data/lib/datadog/core/remote/negotiation.rb

@@ -11,7 +11,7 @@ module Datadog
           transport_options = {}
           transport_options[:agent_settings] = agent_settings if agent_settings
 
-          @transport_root = Datadog::Core::Remote::Transport::HTTP.root(**transport_options.dup)
+          @transport_root = Datadog::Core::Remote::Transport::HTTP.root(**transport_options) # steep:ignore
           @logged = suppress_logging
         end