datadog 2.7.1 → 2.8.0

data/lib/datadog/appsec/contrib/active_record/patcher.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require_relative '../patcher'
+require_relative 'instrumentation'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module ActiveRecord
+        # AppSec patcher module for ActiveRecord
+        module Patcher
+          include Datadog::AppSec::Contrib::Patcher
+
+          module_function
+
+          def patched?
+            Patcher.instance_variable_get(:@patched)
+          end
+
+          def target_version
+            Integration.version
+          end
+
+          def patch
+            ActiveSupport.on_load :active_record do
+              instrumentation_module = if ::ActiveRecord.gem_version >= Gem::Version.new('7.1')
+                                         Instrumentation::InternalExecQueryAdapterPatch
+                                       else
+                                         Instrumentation::ExecQueryAdapterPatch
+                                       end
+
+              if defined?(::ActiveRecord::ConnectionAdapters::SQLite3Adapter)
+                ::ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend(instrumentation_module)
+              end
+
+              if defined?(::ActiveRecord::ConnectionAdapters::Mysql2Adapter)
+                ::ActiveRecord::ConnectionAdapters::Mysql2Adapter.prepend(instrumentation_module)
+              end
+
+              if defined?(::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter)
+                unless defined?(::ActiveRecord::ConnectionAdapters::JdbcAdapter)
+                  instrumentation_module = Instrumentation::ExecuteAndClearAdapterPatch
+                end
+
+                ::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(instrumentation_module)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/event.rb

@@ -142,7 +142,7 @@ module Datadog
           scope.trace.keep! if scope.trace
 
           if scope.service_entry_span
-            scope.service_entry_span.set_tag('appsec.blocked', 'true') if waf_result.actions.include?('block')
+            scope.service_entry_span.set_tag('appsec.blocked', 'true') if waf_result.actions.key?('block_request')
             scope.service_entry_span.set_tag('appsec.event', 'true')
           end
 

data/lib/datadog/appsec/processor/context.rb

@@ -19,7 +19,7 @@ module Datadog
           @events = []
           @run_mutex = Mutex.new
 
-          @libddwaf_debug_tag = "libddwaf:#{WAF::VERSION::STRING}"
+          @libddwaf_debug_tag = "libddwaf:#{WAF::VERSION::STRING} method:ddwaf_run"
         end
 
         def run(persistent_data, ephemeral_data, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
@@ -79,7 +79,7 @@ module Datadog
           @context.run(persistent_data, ephemeral_data, timeout)
         rescue WAF::LibDDWAF::Error => e
           Datadog.logger.debug { "#{@libddwaf_debug_tag} execution error: #{e} backtrace: #{e.backtrace&.first(3)}" }
-          @telemetry.report(e, description: 'libddwaf internal low-level error')
+          @telemetry.report(e, description: 'libddwaf-rb internal low-level error')
 
           [:err_internal, WAF::Result.new(:err_internal, [], 0.0, false, [], [])]
         end

data/lib/datadog/appsec/remote.rb

@@ -67,7 +67,6 @@ module Datadog
             data = []
             overrides = []
             exclusions = []
-            actions = []
 
             repository.contents.each do |content|
               parsed_content = parse_content(content)
@@ -81,7 +80,6 @@ module Datadog
                 overrides << parsed_content['rules_override'] if parsed_content['rules_override']
                 exclusions << parsed_content['exclusions'] if parsed_content['exclusions']
                 custom_rules << parsed_content['custom_rules'] if parsed_content['custom_rules']
-                actions.concat(parsed_content['actions']) if parsed_content['actions']
               end
             end
 
@@ -105,7 +103,7 @@ module Datadog
               telemetry: telemetry
             )
 
-            Datadog::AppSec.reconfigure(ruleset: ruleset, actions: actions, telemetry: telemetry)
+            Datadog::AppSec.reconfigure(ruleset: ruleset, telemetry: telemetry)
           end
 
           [receiver]

data/lib/datadog/appsec/response.rb

@@ -31,19 +31,16 @@ module Datadog
         def negotiate(env, actions)
           # @type var configured_response: Response?
           configured_response = nil
-          actions.each do |action|
+          actions.each do |type, parameters|
             # Need to use next to make steep happy :(
             # I rather use break to stop the execution
             next if configured_response
 
-            action_configuration = AppSec::Processor::Actions.fetch_configuration(action)
-            next unless action_configuration
-
-            configured_response = case action_configuration['type']
+            configured_response = case type
                                   when 'block_request'
-                                    block_response(env, action_configuration['parameters'])
+                                    block_response(env, parameters)
                                   when 'redirect_request'
-                                    redirect_response(env, action_configuration['parameters'])
+                                    redirect_response(env, parameters)
                                   end
           end
 
@@ -90,7 +87,7 @@ module Datadog
           body << content(content_type)
 
           Response.new(
-            status: options['status_code'] || 403,
+            status: options['status_code']&.to_i || 403,
             headers: { 'Content-Type' => content_type },
             body: body,
           )
@@ -100,15 +97,14 @@ module Datadog
           if options['location'] && !options['location'].empty?
             content_type = content_type(env)
 
-            status = options['status_code'] >= 300 && options['status_code'] < 400 ? options['status_code'] : 303
-
             headers = {
               'Content-Type' => content_type,
               'Location' => options['location']
             }
 
+            status_code = options['status_code'].to_i
             Response.new(
-              status: status,
+              status: (status_code >= 300 && status_code < 400 ? status_code : 303),
               headers: headers,
               body: [],
             )

data/lib/datadog/appsec.rb

@@ -24,12 +24,12 @@ module Datadog
         appsec_component.processor if appsec_component
       end
 
-      def reconfigure(ruleset:, actions:, telemetry:)
+      def reconfigure(ruleset:, telemetry:)
         appsec_component = components.appsec
 
         return unless appsec_component
 
-        appsec_component.reconfigure(ruleset: ruleset, actions: actions, telemetry: telemetry)
+        appsec_component.reconfigure(ruleset: ruleset, telemetry: telemetry)
       end
 
       def reconfigure_lock(&block)
@@ -56,6 +56,7 @@ end
 require_relative 'appsec/contrib/rack/integration'
 require_relative 'appsec/contrib/sinatra/integration'
 require_relative 'appsec/contrib/rails/integration'
+require_relative 'appsec/contrib/active_record/integration'
 require_relative 'appsec/contrib/devise/integration'
 require_relative 'appsec/contrib/graphql/integration'
 

data/lib/datadog/core/configuration/components.rb

@@ -13,6 +13,7 @@ require_relative '../remote/component'
 require_relative '../../tracing/component'
 require_relative '../../profiling/component'
 require_relative '../../appsec/component'
+require_relative '../../di/component'
 require_relative '../crashtracking/component'
 
 module Datadog
@@ -83,6 +84,7 @@ module Datadog
           :telemetry,
           :tracer,
           :crashtracker,
+          :dynamic_instrumentation,
           :appsec
 
         def initialize(settings)
@@ -110,12 +112,13 @@ module Datadog
           @runtime_metrics = self.class.build_runtime_metrics_worker(settings)
           @health_metrics = self.class.build_health_metrics(settings)
           @appsec = Datadog::AppSec::Component.build_appsec_component(settings, telemetry: telemetry)
+          @dynamic_instrumentation = Datadog::DI::Component.build(settings, agent_settings, telemetry: telemetry)
 
           self.class.configure_tracing(settings)
         end
 
         # Starts up components
-        def startup!(settings)
+        def startup!(settings, old_state: nil)
           if settings.profiling.enabled
             if profiler
               profiler.start
@@ -126,6 +129,16 @@ module Datadog
             end
           end
 
+          if settings.remote.enabled && old_state&.[](:remote_started)
+            # The library was reconfigured and previously it already started
+            # the remote component (i.e., it received at least one request
+            # through the installed Rack middleware which started the remote).
+            # If the new configuration also has remote enabled, start the
+            # new remote right away.
+            # remote should always be not nil here but steep doesn't know this.
+            remote&.start
+          end
+
           Core::Diagnostics::EnvironmentLogger.collect_and_log!(@environment_logger_extra)
         end
 
@@ -136,6 +149,9 @@ module Datadog
           # Shutdown remote configuration
           remote.shutdown! if remote
 
+          # Shutdown DI after remote, since remote config triggers DI operations.
+          dynamic_instrumentation&.shutdown!
+
           # Decommission AppSec
           appsec.shutdown! if appsec
 

data/lib/datadog/core/configuration/settings.rb

@@ -863,6 +863,16 @@ module Datadog
             o.type :float
             o.default 1.0
           end
+
+          # Enable log collection for telemetry. Log collection only works when telemetry is enabled and
+          # logs are enabled.
+          # @default `DD_TELEMETRY_LOG_COLLECTION_ENABLED` environment variable, otherwise `true`.
+          # @return [Boolean]
+          option :log_collection_enabled do |o|
+            o.type :bool
+            o.env Core::Telemetry::Ext::ENV_LOG_COLLECTION
+            o.default true
+          end
         end
 
         # Remote configuration

data/lib/datadog/core/configuration.rb

@@ -258,8 +258,16 @@ module Datadog
       def replace_components!(settings, old)
         components = Components.new(settings)
 
+        # Carry over state from existing components to the new ones.
+        # Currently, if we already started the remote component (which
+        # happens after a request goes through installed Rack middleware),
+        # we will start the new remote component as well.
+        old_state = {
+          remote_started: old.remote&.started?,
+        }
+
         old.shutdown!(components)
-        components.startup!(settings)
+        components.startup!(settings, old_state: old_state)
         components
       end
 

data/lib/datadog/core/remote/client/capabilities.rb

@@ -32,6 +32,12 @@ module Datadog
               register_receivers(Datadog::AppSec::Remote.receivers(@telemetry))
             end
 
+            if settings.respond_to?(:dynamic_instrumentation) && settings.dynamic_instrumentation.enabled
+              register_capabilities(Datadog::DI::Remote.capabilities)
+              register_products(Datadog::DI::Remote.products)
+              register_receivers(Datadog::DI::Remote.receivers(@telemetry))
+            end
+
             register_capabilities(Datadog::Tracing::Remote.capabilities)
             register_products(Datadog::Tracing::Remote.products)
             register_receivers(Datadog::Tracing::Remote.receivers(@telemetry))

data/lib/datadog/core/remote/client.rb

@@ -24,93 +24,99 @@ module Datadog
           @dispatcher = Dispatcher.new(@capabilities.receivers)
         end
 
-        # rubocop:disable Metrics/AbcSize,Metrics/PerceivedComplexity,Metrics/MethodLength,Metrics/CyclomaticComplexity
         def sync
           # TODO: Skip sync if no capabilities are registered
           response = transport.send_config(payload)
 
           if response.ok?
-            # when response is completely empty, do nothing as in: leave as is
-            if response.empty?
-              Datadog.logger.debug { 'remote: empty response => NOOP' }
+            process_response(response)
+          elsif response.internal_error?
+            raise TransportError, response.to_s
+          end
+        end
 
-              return
-            end
+        private
 
-            begin
-              paths = response.client_configs.map do |path|
-                Configuration::Path.parse(path)
-              end
+        def process_response(response)
+          # when response is completely empty, do nothing as in: leave as is
+          if response.empty?
+            Datadog.logger.debug { 'remote: empty response => NOOP' }
 
-              targets = Configuration::TargetMap.parse(response.targets)
+            return
+          end
 
-              contents = Configuration::ContentList.parse(response.target_files)
-            rescue Remote::Configuration::Path::ParseError => e
-              raise SyncError, e.message
+          begin
+            paths = response.client_configs.map do |path|
+              Configuration::Path.parse(path)
             end
 
-            # To make sure steep does not complain
-            return unless paths && targets && contents
+            targets = Configuration::TargetMap.parse(response.targets)
+
+            contents = Configuration::ContentList.parse(response.target_files)
+          rescue Remote::Configuration::Path::ParseError => e
+            raise SyncError, e.message
+          end
 
-            # TODO: sometimes it can strangely be so that paths.empty?
-            # TODO: sometimes it can strangely be so that targets.empty?
+          # To make sure steep does not complain
+          return unless paths && targets && contents
 
-            changes = repository.transaction do |current, transaction|
-              # paths to be removed: previously applied paths minus ingress paths
-              (current.paths - paths).each { |p| transaction.delete(p) }
+          # TODO: sometimes it can strangely be so that paths.empty?
+          # TODO: sometimes it can strangely be so that targets.empty?
 
-              # go through each ingress path
-              paths.each do |path|
-                # match target with path
-                target = targets[path]
+          apply_config(paths, targets, contents)
+        end
 
-                # abort entirely if matching target not found
-                raise SyncError, "no target for path '#{path}'" if target.nil?
+        def apply_config(paths, targets, contents)
+          changes = repository.transaction do |current, transaction|
+            # paths to be removed: previously applied paths minus ingress paths
+            (current.paths - paths).each { |p| transaction.delete(p) }
 
-                # new paths are not in previously applied paths
-                new = !current.paths.include?(path)
+            # go through each ingress path
+            paths.each do |path|
+              # match target with path
+              target = targets[path]
 
-                # updated paths are in previously applied paths
-                # but the content hash changed
-                changed = current.paths.include?(path) && !current.contents.find_content(path, target)
+              # abort entirely if matching target not found
+              raise SyncError, "no target for path '#{path}'" if target.nil?
 
-                # skip if unchanged
-                same = !new && !changed
+              # new paths are not in previously applied paths
+              new = !current.paths.include?(path)
 
-                next if same
+              # updated paths are in previously applied paths
+              # but the content hash changed
+              changed = current.paths.include?(path) && !current.contents.find_content(path, target)
 
-                # match content with path and target
-                content = contents.find_content(path, target)
+              # skip if unchanged
+              same = !new && !changed
 
-                # abort entirely if matching content not found
-                raise SyncError, "no valid content for target at path '#{path}'" if content.nil?
+              next if same
 
-                # to be added or updated << config
-                # TODO: metadata (hash, version, etc...)
-                transaction.insert(path, target, content) if new
-                transaction.update(path, target, content) if changed
-              end
+              # match content with path and target
+              content = contents.find_content(path, target)
 
-              # save backend opaque backend state
-              transaction.set(opaque_backend_state: targets.opaque_backend_state)
-              transaction.set(targets_version: targets.version)
+              # abort entirely if matching content not found
+              raise SyncError, "no valid content for target at path '#{path}'" if content.nil?
 
-              # upon transaction end, new list of applied config + metadata (add, change, remove) will be saved
-              # TODO: also remove stale config (matching removed) from cache (client configs is exhaustive list of paths)
+              # to be added or updated << config
+              # TODO: metadata (hash, version, etc...)
+              transaction.insert(path, target, content) if new
+              transaction.update(path, target, content) if changed
             end
 
-            if changes.empty?
-              Datadog.logger.debug { 'remote: no changes' }
-            else
-              dispatcher.dispatch(changes, repository)
-            end
-          elsif response.internal_error?
-            raise TransportError, response.to_s
+            # save backend opaque backend state
+            transaction.set(opaque_backend_state: targets.opaque_backend_state)
+            transaction.set(targets_version: targets.version)
+
+            # upon transaction end, new list of applied config + metadata (add, change, remove) will be saved
+            # TODO: also remove stale config (matching removed) from cache (client configs is exhaustive list of paths)
           end
-        end
-        # rubocop:enable Metrics/AbcSize,Metrics/PerceivedComplexity,Metrics/MethodLength,Metrics/CyclomaticComplexity
 
-        private
+          if changes.empty?
+            Datadog.logger.debug { 'remote: no changes' }
+          else
+            dispatcher.dispatch(changes, repository)
+          end
+        end
 
         def payload # rubocop:disable Metrics/MethodLength
           state = repository.state

data/lib/datadog/core/telemetry/component.rb

@@ -14,6 +14,7 @@ module Datadog
   module Core
     module Telemetry
       # Telemetry entrypoint, coordinates sending telemetry events at various points in app lifecycle.
+      # Note: Telemetry does not spawn its worker thread in fork processes, thus no telemetry is sent in forked processes.
       class Component
         attr_reader :enabled
 
@@ -52,6 +53,7 @@ module Datadog
             metrics_aggregation_interval_seconds: settings.telemetry.metrics_aggregation_interval_seconds,
             dependency_collection: settings.telemetry.dependency_collection,
             shutdown_timeout_seconds: settings.telemetry.shutdown_timeout_seconds,
+            log_collection_enabled: settings.telemetry.log_collection_enabled
           )
         end
 
@@ -67,10 +69,11 @@ module Datadog
           http_transport:,
           shutdown_timeout_seconds:,
           enabled: true,
-          metrics_enabled: true
+          metrics_enabled: true,
+          log_collection_enabled: true
         )
           @enabled = enabled
-          @stopped = false
+          @log_collection_enabled = log_collection_enabled
 
           @metrics_manager = MetricsManager.new(
             enabled: enabled && metrics_enabled,
@@ -86,6 +89,9 @@ module Datadog
             dependency_collection: dependency_collection,
             shutdown_timeout: shutdown_timeout_seconds
           )
+
+          @stopped = false
+
           @worker.start
         end
 
@@ -114,7 +120,7 @@ module Datadog
         end
 
         def log!(event)
-          return unless @enabled || forked?
+          return if !@enabled || forked? || !@log_collection_enabled
 
           @worker.enqueue(event)
         end

data/lib/datadog/core/telemetry/ext.rb

@@ -13,6 +13,7 @@ module Datadog
         ENV_INSTALL_TYPE = 'DD_INSTRUMENTATION_INSTALL_TYPE'
         ENV_INSTALL_TIME = 'DD_INSTRUMENTATION_INSTALL_TIME'
         ENV_AGENTLESS_URL_OVERRIDE = 'DD_TELEMETRY_AGENTLESS_URL'
+        ENV_LOG_COLLECTION = 'DD_TELEMETRY_LOG_COLLECTION_ENABLED'
       end
     end
   end

data/lib/datadog/di/code_tracker.rb

@@ -78,17 +78,18 @@ module Datadog
               registry_lock.synchronize do
                 registry[path] = tp.instruction_sequence
               end
-            end
-
-            DI.component&.probe_manager&.install_pending_line_probes(path)
 
+              # Also, pending line probes should only be installed for
+              # non-eval'd code.
+              DI.current_component&.probe_manager&.install_pending_line_probes(path)
+            end
           # Since this method normally is called from customer applications,
           # rescue any exceptions that might not be handled to not break said
           # customer applications.
           rescue => exc
             # TODO we do not have DI.component defined yet, remove steep:ignore
             # before release.
-            if component = DI.component # steep:ignore
+            if component = DI.current_component # steep:ignore
               raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
               component.logger.warn("Unhandled exception in script_compiled trace point: #{exc.class}: #{exc}")
               component.telemetry&.report(exc, description: "Unhandled exception in script_compiled trace point")

data/lib/datadog/di/component.rb

@@ -24,7 +24,9 @@ module Datadog
 
           return unless environment_supported?(settings)
 
-          new(settings, agent_settings, Datadog.logger, code_tracker: DI.code_tracker, telemetry: telemetry)
+          new(settings, agent_settings, Datadog.logger, code_tracker: DI.code_tracker, telemetry: telemetry).tap do |component|
+            DI.add_current_component(component)
+          end
         end
 
         def build!(settings, agent_settings, telemetry: nil)
@@ -99,6 +101,8 @@ module Datadog
       # was replaced by a new instance, the new instance of it wouldn't have
       # any of the already loaded code tracked.
       def shutdown!(replacement = nil)
+        DI.remove_current_component(self)
+
         probe_manager.clear_hooks
         probe_manager.close
         probe_notifier_worker.stop

data/lib/datadog/di/contrib/active_record.rb

@@ -5,6 +5,7 @@ Datadog::DI::Serializer.register(condition: lambda { |value| ActiveRecord::Base
   # steep:ignore:start
   value_to_serialize = {
     attributes: value.attributes,
+    new_record: value.new_record?,
   }
   serializer.serialize_value(value_to_serialize, depth: depth ? depth - 1 : nil, type: value.class)
   # steep:ignore:end

data/lib/datadog/di/init.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Require 'datadog/di/init' early in the application boot process to
+# enable dynamic instrumentation for third-party libraries used by the
+# application.
+
+require_relative '../tracing'
+require_relative '../tracing/contrib'
+require_relative '../di'
+
+# Code tracking is required for line probes to work; see the comments
+# on the activate_tracking methods in di.rb for further details.
+#
+# Unlike di.rb which conditionally activates tracking only if the
+# DD_DYNAMIC_INSTRUMENTATION_ENABLED environment variable is set, this file
+# always activates tracking. This is because this file is explicitly loaded
+# by customer applications for the purpose of enabling code tracking
+# early in application boot process (i.e., before datadog library itself
+# is loaded).
+Datadog::DI.activate_tracking