datadog 2.9.0 → 2.10.0

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -24,7 +24,7 @@ module Datadog
                 engine = AppSec::Reactive::Engine.new
 
                 Monitor::Reactive::SetUser.subscribe(engine, context) do |result|
-                  if result.status == :match
+                  if result.match?
                     # TODO: should this hash be an Event instance instead?
                     event = {
                       waf_result: result,
@@ -37,12 +37,13 @@ module Datadog
                     # We want to keep the trace in case of security event
                     context.trace.keep! if context.trace
                     Datadog::AppSec::Event.tag_and_keep!(context, result)
-                    context.waf_runner.events << event
+                    context.events << event
+
+                    Datadog::AppSec::ActionsHandler.handle(result.actions)
                   end
                 end
 
-                block = Monitor::Reactive::SetUser.publish(engine, user)
-                throw(Datadog::AppSec::Ext::INTERRUPT, event[:actions]) if block
+                Monitor::Reactive::SetUser.publish(engine, user)
 
                 stack.call(user)
               end

data/lib/datadog/appsec/monitor/reactive/set_user.rb

@@ -32,7 +32,7 @@ module Datadog
               waf_timeout = Datadog.configuration.appsec.waf_timeout
               result = context.run_waf(persistent_data, {}, waf_timeout)
 
-              next if result.status != :match
+              next unless result.match?
 
               yield result
               throw(:block, true) unless result.actions.empty?

data/lib/datadog/appsec/processor.rb

@@ -1,10 +1,11 @@
 # frozen_string_literal: true
 
-require_relative 'processor/context'
+require_relative 'security_engine/runner'
 
 module Datadog
   module AppSec
     # Processor integrates libddwaf into datadog/appsec
+    # NOTE: This class will be moved under AppSec::SecurityEngine namespace
     class Processor
       attr_reader :diagnostics, :addresses
 
@@ -29,8 +30,8 @@ module Datadog
         @handle.finalize
       end
 
-      def new_context
-        Context.new(@handle, telemetry: @telemetry)
+      def new_runner
+        SecurityEngine::Runner.new(@handle, telemetry: @telemetry)
       end
 
       private

data/lib/datadog/appsec/response.rb

@@ -19,100 +19,38 @@ module Datadog
         [status, headers, body]
       end
 
-      def to_sinatra_response
-        ::Sinatra::Response.new(body, status, headers)
-      end
-
-      def to_action_dispatch_response
-        ::ActionDispatch::Response.new(status, headers, body)
-      end
-
       class << self
-        def negotiate(env, actions)
-          # @type var configured_response: Response?
-          configured_response = nil
-          actions.each do |type, parameters|
-            # Need to use next to make steep happy :(
-            # I rather use break to stop the execution
-            next if configured_response
-
-            configured_response = case type
-                                  when 'block_request'
-                                    block_response(env, parameters)
-                                  when 'redirect_request'
-                                    redirect_response(env, parameters)
-                                  end
-          end
-
-          configured_response || default_response(env)
-        end
-
-        def graphql_response(gateway_multiplex)
-          multiplex_return = []
-          gateway_multiplex.queries.each do |query|
-            # This method is only called in places where GraphQL-Ruby is already required
-            query_result = ::GraphQL::Query::Result.new(
-              query: query,
-              values: JSON.parse(content('application/json'))
-            )
-            multiplex_return << query_result
-          end
+        def from_interrupt_params(interrupt_params, http_accept_header)
+          return redirect_response(interrupt_params) if interrupt_params['location']
 
-          multiplex_return
+          block_response(interrupt_params, http_accept_header)
         end
 
         private
 
-        def default_response(env)
-          content_type = content_type(env)
-
-          body = []
-          body << content(content_type)
+        def block_response(interrupt_params, http_accept_header)
+          content_type = case interrupt_params['type']
+                         when nil, 'auto' then content_type(http_accept_header)
+                         else FORMAT_TO_CONTENT_TYPE.fetch(interrupt_params['type'], DEFAULT_CONTENT_TYPE)
+                         end
 
           Response.new(
-            status: 403,
+            status: interrupt_params['status_code']&.to_i || 403,
             headers: { 'Content-Type' => content_type },
-            body: body,
+            body: [content(content_type)],
           )
         end
 
-        def block_response(env, options)
-          content_type = if options['type'] == 'auto'
-                           content_type(env)
-                         else
-                           FORMAT_TO_CONTENT_TYPE[options['type']]
-                         end
-
-          body = []
-          body << content(content_type)
+        def redirect_response(interrupt_params)
+          status_code = interrupt_params['status_code'].to_i
 
           Response.new(
-            status: options['status_code']&.to_i || 403,
-            headers: { 'Content-Type' => content_type },
-            body: body,
+            status: (status_code >= 300 && status_code < 400 ? status_code : 303),
+            headers: { 'Location' => interrupt_params.fetch('location') },
+            body: [],
           )
         end
 
-        def redirect_response(env, options)
-          if options['location'] && !options['location'].empty?
-            content_type = content_type(env)
-
-            headers = {
-              'Content-Type' => content_type,
-              'Location' => options['location']
-            }
-
-            status_code = options['status_code'].to_i
-            Response.new(
-              status: (status_code >= 300 && status_code < 400 ? status_code : 303),
-              headers: headers,
-              body: [],
-            )
-          else
-            default_response(env)
-          end
-        end
-
         CONTENT_TYPE_TO_FORMAT = {
           'application/json' => :json,
           'text/html' => :html,
@@ -126,10 +64,10 @@ module Datadog
 
         DEFAULT_CONTENT_TYPE = 'application/json'
 
-        def content_type(env)
-          return DEFAULT_CONTENT_TYPE unless env.key?('HTTP_ACCEPT')
+        def content_type(http_accept_header)
+          return DEFAULT_CONTENT_TYPE if http_accept_header.nil?
 
-          accept_types = env['HTTP_ACCEPT'].split(',').map(&:strip)
+          accept_types = http_accept_header.split(',').map(&:strip)
 
           accepted = accept_types.map { |m| Utils::HTTP::MediaRange.new(m) }.sort!.reverse!
 

data/lib/datadog/appsec/security_engine/result.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module SecurityEngine
+      # A namespace for value-objects representing the result of WAF check.
+      module Result
+        # A generic result without indication of its type.
+        class Base
+          attr_reader :events, :actions, :derivatives, :duration_ns, :duration_ext_ns
+
+          def initialize(events:, actions:, derivatives:, timeout:, duration_ns:, duration_ext_ns:)
+            @events = events
+            @actions = actions
+            @derivatives = derivatives
+
+            @timeout = timeout
+            @duration_ns = duration_ns
+            @duration_ext_ns = duration_ext_ns
+          end
+
+          def timeout?
+            !!@timeout
+          end
+
+          def match?
+            raise NotImplementedError
+          end
+        end
+
+        # A result that indicates a security rule match
+        class Match < Base
+          def match?
+            true
+          end
+        end
+
+        # A result that indicates a successful security rules check without a match
+        class Ok < Base
+          def match?
+            false
+          end
+        end
+
+        # A result that indicates an internal security library error
+        class Error
+          attr_reader :events, :actions, :derivatives, :duration_ns, :duration_ext_ns
+
+          def initialize(duration_ext_ns:)
+            @events = []
+            @actions = @derivatives = {}
+            @duration_ns = 0
+            @duration_ext_ns = duration_ext_ns
+          end
+
+          def timeout?
+            false
+          end
+
+          def match?
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/security_engine/runner.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require_relative 'result'
+
+module Datadog
+  module AppSec
+    module SecurityEngine
+      # A class that check input via security engine (WAF) and respond with result.
+      class Runner
+        SUCCESSFUL_EXECUTION_CODES = [:ok, :match].freeze
+
+        def initialize(handle, telemetry:)
+          @mutex = Mutex.new
+          @context = WAF::Context.new(handle)
+          @telemetry = telemetry
+
+          @debug_tag = "libddwaf:#{WAF::VERSION::STRING} method:ddwaf_run"
+        end
+
+        def run(persistent_data, ephemeral_data, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
+          @mutex.lock
+
+          start_ns = Core::Utils::Time.get_time(:nanosecond)
+          persistent_data.reject! do |_, v|
+            next false if v.is_a?(TrueClass) || v.is_a?(FalseClass)
+
+            v.nil? ? true : v.empty?
+          end
+
+          ephemeral_data.reject! do |_, v|
+            next false if v.is_a?(TrueClass) || v.is_a?(FalseClass)
+
+            v.nil? ? true : v.empty?
+          end
+
+          _code, result = try_run(persistent_data, ephemeral_data, timeout)
+          stop_ns = Core::Utils::Time.get_time(:nanosecond)
+
+          report_execution(result)
+
+          unless SUCCESSFUL_EXECUTION_CODES.include?(result.status)
+            return Result::Error.new(duration_ext_ns: stop_ns - start_ns)
+          end
+
+          klass = result.status == :match ? Result::Match : Result::Ok
+          klass.new(
+            events: result.events,
+            actions: result.actions,
+            derivatives: result.derivatives,
+            timeout: result.timeout,
+            duration_ns: result.total_runtime,
+            duration_ext_ns: (stop_ns - start_ns)
+          )
+        ensure
+          @mutex.unlock
+        end
+
+        def finalize
+          @context.finalize
+        end
+
+        private
+
+        def try_run(persistent_data, ephemeral_data, timeout)
+          @context.run(persistent_data, ephemeral_data, timeout)
+        rescue WAF::LibDDWAF::Error => e
+          Datadog.logger.debug { "#{@debug_tag} execution error: #{e} backtrace: #{e.backtrace&.first(3)}" }
+          @telemetry.report(e, description: 'libddwaf-rb internal low-level error')
+
+          [:err_internal, WAF::Result.new(:err_internal, [], 0, false, [], [])]
+        end
+
+        def report_execution(result)
+          Datadog.logger.debug { "#{@debug_tag} execution timed out: #{result.inspect}" } if result.timeout
+
+          if SUCCESSFUL_EXECUTION_CODES.include?(result.status)
+            Datadog.logger.debug { "#{@debug_tag} execution result: #{result.inspect}" }
+          else
+            message = "#{@debug_tag} execution error: #{result.status.inspect}"
+
+            Datadog.logger.debug { message }
+            @telemetry.error(message)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/security_engine.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # A namespace for secutiry library we use to detect and prevent threats.
+    module SecurityEngine
+    end
+  end
+end

data/lib/datadog/appsec.rb

@@ -14,19 +14,24 @@ module Datadog
         Datadog.configuration.appsec.enabled
       end
 
+      def rasp_enabled?
+        Datadog.configuration.appsec.rasp_enabled
+      end
+
       def active_context
         Datadog::AppSec::Context.active
       end
 
-      def processor
-        appsec_component = components.appsec
+      def telemetry
+        components.appsec&.telemetry
+      end
 
-        appsec_component.processor if appsec_component
+      def processor
+        components.appsec&.processor
       end
 
       def reconfigure(ruleset:, telemetry:)
         appsec_component = components.appsec
-
         return unless appsec_component
 
         appsec_component.reconfigure(ruleset: ruleset, telemetry: telemetry)
@@ -34,12 +39,16 @@ module Datadog
 
       def reconfigure_lock(&block)
         appsec_component = components.appsec
-
         return unless appsec_component
 
         appsec_component.reconfigure_lock(&block)
       end
 
+      def api_security_enabled?
+        Datadog.configuration.appsec.api_security.enabled &&
+          Datadog.configuration.appsec.api_security.sample_rate.sample?
+      end
+
       private
 
       def components

data/lib/datadog/di/component.rb

@@ -76,6 +76,7 @@ module Datadog
         @agent_settings = agent_settings
         @logger = logger
         @telemetry = telemetry
+        @code_tracker = code_tracker
         @redactor = Redactor.new(settings)
         @serializer = Serializer.new(settings, redactor, telemetry: telemetry)
         @instrumenter = Instrumenter.new(settings, serializer, logger, code_tracker: code_tracker, telemetry: telemetry)
@@ -90,6 +91,7 @@ module Datadog
       attr_reader :agent_settings
       attr_reader :logger
       attr_reader :telemetry
+      attr_reader :code_tracker
       attr_reader :instrumenter
       attr_reader :transport
       attr_reader :probe_notifier_worker

data/lib/datadog/di/probe_notification_builder.rb

@@ -32,6 +32,12 @@ module Datadog
           status: 'EMITTING',)
       end
 
+      def build_errored(probe, exc)
+        build_status(probe,
+          message: "Instrumentation for probe #{probe.id} failed: #{exc}",
+          status: 'ERROR',)
+      end
+
       # Duration is in seconds.
       def build_executed(probe,
         trace_point: nil, rv: nil, duration: nil, caller_locations: nil,

data/lib/datadog/di/redactor.rb

@@ -119,7 +119,6 @@ module Datadog
         "dburl",
         "encryptionkey",
         "encryptionkeyid",
-        "env",
         "geolocation",
         "gpgkey",
         "ipaddress",

data/lib/datadog/di/remote.rb

@@ -49,28 +49,48 @@ module Datadog
                   begin
                     probe_spec = parse_content(content)
                     probe = ProbeBuilder.build_from_remote_config(probe_spec)
-                    payload = component.probe_notification_builder.build_received(probe)
-                    component.probe_notifier_worker.add_status(payload)
+                    probe_notification_builder = component.probe_notification_builder
+                    payload = probe_notification_builder.build_received(probe)
+                    probe_notifier_worker = component.probe_notifier_worker
+                    probe_notifier_worker.add_status(payload)
                     component.logger.debug { "di: received probe from RC: #{probe.type} #{probe.location}" }
 
                     begin
                       # TODO test exception capture
                       probe_manager.add_probe(probe)
                       content.applied
+                    rescue DI::Error::DITargetNotInRegistry => exc
+                      component.telemetry&.report(exc, description: "Line probe is targeting a loaded file that is not in code tracker")
+
+                      payload = probe_notification_builder.build_errored(probe, exc)
+                      probe_notifier_worker.add_status(payload)
+
+                      # If a probe fails to install, we will mark the content
+                      # as errored. On subsequent remote configuration application
+                      # attemps, probe manager will raise the "previously errored"
+                      # exception and we'll rescue it here, again marking the
+                      # content as errored but with a somewhat different exception
+                      # message.
+                      # TODO assert content state (errored for this example)
+                      content.errored("Error applying dynamic instrumentation configuration: #{exc.class.name} #{exc.message}")
                     rescue => exc
                       raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
 
                       component.logger.debug { "di: unhandled exception adding probe in DI remote receiver: #{exc.class}: #{exc}" }
                       component.telemetry&.report(exc, description: "Unhandled exception adding probe in DI remote receiver")
 
+                      # TODO test this path
+                      payload = probe_notification_builder.build_errored(probe, exc)
+                      probe_notifier_worker.add_status(payload)
+
                       # If a probe fails to install, we will mark the content
                       # as errored. On subsequent remote configuration application
                       # attemps, probe manager will raise the "previously errored"
                       # exception and we'll rescue it here, again marking the
                       # content as errored but with a somewhat different exception
                       # message.
-                      # TODO stack trace must be redacted or not sent at all
-                      content.errored("Error applying dynamic instrumentation configuration: #{exc.class.name} #{exc.message}: #{Array(exc.backtrace).join("\n")}")
+                      # TODO assert content state (errored for this example)
+                      content.errored("Error applying dynamic instrumentation configuration: #{exc.class.name} #{exc.message}")
                     end
 
                     # Important: even if processing fails for this probe config,
@@ -84,7 +104,8 @@ module Datadog
                     component.logger.debug { "di: unhandled exception handling probe in DI remote receiver: #{exc.class}: #{exc}" }
                     component.telemetry&.report(exc, description: "Unhandled exception handling probe in DI remote receiver")
 
-                    content.errored("Error applying dynamic instrumentation configuration: #{exc.class.name} #{exc.message}: #{Array(exc.backtrace).join("\n")}")
+                    # TODO assert content state (errored for this example)
+                    content.errored("Error applying dynamic instrumentation configuration: #{exc.class.name} #{exc.message}")
                   end
                 end
               end

data/lib/datadog/tracing/contrib/aws/integration.rb

@@ -17,7 +17,7 @@ module Datadog
           # @public_api Changing the integration name or integration options can cause breaking changes
           register_as :aws, auto_patch: true
           def self.gem_name
-            'aws-sdk-core'
+            'aws-sdk'
           end
 
           def self.version

data/lib/datadog/tracing/contrib/extensions.rb

@@ -110,6 +110,13 @@ module Datadog
           module Settings
             InvalidIntegrationError = Class.new(StandardError)
 
+            # Used to avoid concurrency issues between registering integrations (e.g. mutation) and reporting the
+            # current integrations for logging/debugging/telemetry purposes (e.g. iteration) in the
+            # `@instrumented_integrations` hash.
+            #
+            # See https://github.com/DataDog/dd-trace-rb/issues/2851 for details on the original issue.
+            INSTRUMENTED_INTEGRATIONS_LOCK = Mutex.new
+
             def self.included(base)
               base.class_eval do
                 settings :contrib do
@@ -161,7 +168,10 @@ module Datadog
                 configuration_name = options[:describes] || :default
                 filtered_options = options.reject { |k, _v| k == :describes }
                 integration.configure(configuration_name, filtered_options, &block)
-                instrumented_integrations[integration_name] = integration
+                INSTRUMENTED_INTEGRATIONS_LOCK.synchronize do
+                  @instrumented_integrations ||= {}
+                  @instrumented_integrations[integration_name] = integration
+                end
 
                 # Add to activation list
                 integrations_pending_activation << integration
@@ -192,14 +202,16 @@ module Datadog
               @integrations_pending_activation ||= Set.new
             end
 
+            # This method is only for logging/debugging/telemetry purposes (e.g. iteration) in the
+            # `@instrumented_integrations` hash.
             # @!visibility private
             def instrumented_integrations
-              @instrumented_integrations ||= {}
+              INSTRUMENTED_INTEGRATIONS_LOCK.synchronize { (@instrumented_integrations&.dup || {}).freeze }
             end
 
             # @!visibility private
             def reset!
-              instrumented_integrations.clear
+              INSTRUMENTED_INTEGRATIONS_LOCK.synchronize { @instrumented_integrations&.clear }
               super
             end
 

data/lib/datadog/tracing/contrib/http/integration.rb

@@ -22,6 +22,9 @@ module Datadog
 
           # @public_api Changing the integration name or integration options can cause breaking changes
           register_as :http, auto_patch: true
+          def self.gem_name
+            'net-http'
+          end
 
           def self.version
             Gem::Version.new(RUBY_VERSION)

data/lib/datadog/version.rb

@@ -3,7 +3,7 @@
 module Datadog
   module VERSION
     MAJOR = 2
-    MINOR = 9
+    MINOR = 10
     PATCH = 0
     PRE = nil
     BUILD = nil