datadog 2.19.0 → 2.20.0

data/lib/datadog/appsec/event.rb

@@ -2,6 +2,7 @@
 
 require 'json'
 require_relative 'rate_limiter'
+require_relative 'trace_keeper'
 require_relative 'compressed_json'
 
 module Datadog
@@ -40,8 +41,7 @@ module Datadog
 
       class << self
         def tag_and_keep!(context, waf_result)
-          # We want to keep the trace in case of security event
-          context.trace&.keep!
+          TraceKeeper.keep!(context.trace)
 
           if context.span
             if waf_result.actions.key?('block_request') || waf_result.actions.key?('redirect_request')
@@ -50,8 +50,6 @@ module Datadog
 
             context.span.set_tag('appsec.event', 'true')
           end
-
-          add_distributed_tags(context.trace)
         end
 
         def record(context, request: nil, response: nil)
@@ -66,8 +64,7 @@ module Datadog
               end
 
               if event_group.any? { |event| event.attack? || event.schema? }
-                trace.keep!
-                trace[Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER] = Tracing::Sampling::Ext::Decision::ASM
+                TraceKeeper.keep!(trace)
 
                 context.span['_dd.origin'] = 'appsec'
                 context.span.set_tags(request_tags(request)) if request
@@ -138,18 +135,6 @@ module Datadog
 
           nil
         end
-
-        # Propagate to downstream services the information that the current distributed trace is
-        # containing at least one ASM security event.
-        def add_distributed_tags(trace)
-          return unless trace
-
-          trace.set_tag(
-            Datadog::Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
-            Datadog::Tracing::Sampling::Ext::Decision::ASM
-          )
-          trace.set_distributed_source(Datadog::AppSec::Ext::PRODUCT_BIT)
-        end
       end
     end
   end

data/lib/datadog/appsec/instrumentation/gateway/argument.rb

@@ -21,6 +21,22 @@ module Datadog
             @session_id = session_id
           end
         end
+
+        # This class is used to pass arbitrary data to the event system with an
+        # option to tie it to a context.
+        #
+        # NOTE: This class is a subject of elimination and will be removed when
+        #       the event system is refactored.
+        class DataContainer < Argument
+          attr_reader :data, :context
+
+          def initialize(data, context:)
+            super()
+
+            @data = data
+            @context = context
+          end
+        end
       end
     end
   end

data/lib/datadog/appsec/metrics/collector.rb

@@ -5,19 +5,21 @@ module Datadog
     module Metrics
       # A class responsible for collecting WAF and RASP call metrics.
       class Collector
-        Store = Struct.new(:evals, :timeouts, :duration_ns, :duration_ext_ns, keyword_init: true)
+        Store = Struct.new(:evals, :matches, :errors, :timeouts, :duration_ns, :duration_ext_ns, keyword_init: true)
 
         attr_reader :waf, :rasp
 
         def initialize
           @mutex = Mutex.new
-          @waf = Store.new(evals: 0, timeouts: 0, duration_ns: 0, duration_ext_ns: 0)
-          @rasp = Store.new(evals: 0, timeouts: 0, duration_ns: 0, duration_ext_ns: 0)
+          @waf = Store.new(evals: 0, matches: 0, errors: 0, timeouts: 0, duration_ns: 0, duration_ext_ns: 0)
+          @rasp = Store.new(evals: 0, matches: 0, errors: 0, timeouts: 0, duration_ns: 0, duration_ext_ns: 0)
         end
 
         def record_waf(result)
           @mutex.synchronize do
             @waf.evals += 1
+            @waf.matches += 1 if result.match?
+            @waf.errors += 1 if result.error?
             @waf.timeouts += 1 if result.timeout?
             @waf.duration_ns += result.duration_ns
             @waf.duration_ext_ns += result.duration_ext_ns
@@ -27,6 +29,8 @@ module Datadog
         def record_rasp(result)
           @mutex.synchronize do
             @rasp.evals += 1
+            @waf.matches += 1 if result.match?
+            @waf.errors += 1 if result.error?
             @rasp.timeouts += 1 if result.timeout?
             @rasp.duration_ns += result.duration_ns
             @rasp.duration_ext_ns += result.duration_ext_ns

data/lib/datadog/appsec/metrics/telemetry.rb

@@ -8,7 +8,7 @@ module Datadog
         module_function
 
         def report_rasp(type, result)
-          return if result.is_a?(SecurityEngine::Result::Error)
+          return if result.error?
 
           tags = {rule_type: type, waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING}
           namespace = Ext::TELEMETRY_METRICS_NAMESPACE

data/lib/datadog/appsec/metrics/telemetry_exporter.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Metrics
+      # A class responsible for exporting WAF request metrics via Telemetry.
+      module TelemetryExporter
+        module_function
+
+        def export_waf_request_metrics(metrics, context)
+          AppSec.telemetry.inc(
+            Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.requests', 1,
+            tags: {
+              waf_version: WAF::VERSION::BASE_STRING,
+              event_rules_version: context.waf_runner_ruleset_version,
+              rule_triggered: metrics.matches.positive?.to_s,
+              waf_error: metrics.errors.positive?.to_s,
+              waf_timeout: metrics.timeouts.positive?.to_s,
+              request_blocked: context.interrupted?.to_s,
+              block_failure: 'false',
+              rate_limited: (!context.trace.sampled?).to_s
+            }
+          )
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/metrics.rb

@@ -11,3 +11,4 @@ end
 require_relative 'metrics/collector'
 require_relative 'metrics/exporter'
 require_relative 'metrics/telemetry'
+require_relative 'metrics/telemetry_exporter'

data/lib/datadog/appsec/security_engine/engine.rb

@@ -20,8 +20,6 @@ module Datadog
           exclusion_data
         ].freeze
 
-        attr_reader :waf_addresses, :ruleset_version
-
         def initialize(appsec_settings:, telemetry:)
           @default_ruleset = appsec_settings.ruleset
 
@@ -39,9 +37,9 @@ module Datadog
 
           diagnostics = load_default_config(telemetry: telemetry)
           report_configuration_diagnostics(diagnostics, action: 'init', telemetry: telemetry)
+          @ruleset_version = diagnostics['ruleset_version']
 
-          @waf_handle = @waf_builder.build_handle
-          @waf_addresses = @waf_handle.known_addresses
+          @handle_ref = ThreadSafeRef.new(@waf_builder.build_handle)
         rescue WAF::Error => e
           error_message = "AppSec security engine failed to initialize"
 
@@ -51,16 +49,8 @@ module Datadog
           raise e
         end
 
-        def finalize!
-          @waf_handle&.finalize!
-          @waf_builder&.finalize!
-
-          @waf_addresses = []
-          @ruleset_version = nil
-        end
-
         def new_runner
-          SecurityEngine::Runner.new(@waf_handle.build_context)
+          SecurityEngine::Runner.new(@handle_ref, ruleset_version: @ruleset_version)
         end
 
         def add_or_update_config(config, path:)
@@ -70,7 +60,7 @@ module Datadog
           remove_config_at_path(DEFAULT_RULES_CONFIG_PATH) if @is_ruleset_update
 
           diagnostics = @waf_builder.add_or_update_config(config, path: path)
-          @ruleset_version = diagnostics['ruleset_version'] if diagnostics.key?('ruleset_version')
+          @reconfigured_ruleset_version = diagnostics['ruleset_version'] if diagnostics.key?('ruleset_version')
           report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
 
           # we need to load default config if diagnostics contains top-level error for rules or processors
@@ -79,6 +69,7 @@ module Datadog
               diagnostics.dig('rules', 'error') ||
               diagnostics.dig('processors', 'errors'))
             diagnostics = load_default_config(telemetry: AppSec.telemetry)
+            @reconfigured_ruleset_version = diagnostics['ruleset_version']
             report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
           end
 
@@ -95,6 +86,7 @@ module Datadog
 
           if result && path != DEFAULT_RULES_CONFIG_PATH && path.include?('ASM_DD')
             diagnostics = load_default_config(telemetry: AppSec.telemetry)
+            @reconfigured_ruleset_version = diagnostics['ruleset_version']
             report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
           end
 
@@ -107,24 +99,17 @@ module Datadog
         end
 
         def reconfigure!
-          old_waf_handle = @waf_handle
-
-          @waf_handle = @waf_builder.build_handle
-          @waf_addresses = @waf_handle.known_addresses
+          new_waf_handle = @waf_builder.build_handle
+          @ruleset_version = @reconfigured_ruleset_version
 
-          old_waf_handle&.finalize!
+          @handle_ref.current = new_waf_handle
         rescue WAF::Error => e
-          error_message = "AppSec security engine failed to reconfigure"
+          # WAF::Error can only be raised during new WAF handle creation or when reading known addresses.
+          # This means that the current WAF handle was not yet substituted.
+          error_message = "AppSec security engine failed to reconfigure, reverting to the previous configuration"
 
           Datadog.logger.error("#{error_message}, error #{e.inspect}")
           AppSec.telemetry.report(e, description: error_message)
-
-          if old_waf_handle
-            Datadog.logger.warn("Reverting to the previous configuration")
-
-            @waf_handle = old_waf_handle
-            @waf_addresses = old_waf_handle.known_addresses
-          end
         end
 
         private
@@ -141,10 +126,7 @@ module Datadog
           # deprecated - ip passlist should be configured via RC
           config['exclusions'] ||= AppSec::Processor::RuleLoader.load_exclusions(ip_passlist: @default_ip_passlist)
 
-          diagnostics = @waf_builder.add_or_update_config(config, path: DEFAULT_RULES_CONFIG_PATH)
-          @ruleset_version = diagnostics['ruleset_version']
-
-          diagnostics
+          @waf_builder.add_or_update_config(config, path: DEFAULT_RULES_CONFIG_PATH)
         end
 
         def report_configuration_diagnostics(diagnostics, action:, telemetry:)
@@ -152,7 +134,7 @@ module Datadog
 
           common_tags = {
             waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING,
-            event_rules_version: diagnostics.fetch('ruleset_version', @ruleset_version).to_s,
+            event_rules_version: diagnostics['ruleset_version'].to_s,
             action: action
           }
 

data/lib/datadog/appsec/security_engine/result.rb

@@ -26,6 +26,10 @@ module Datadog
           def match?
             raise NotImplementedError
           end
+
+          def error?
+            raise NotImplementedError
+          end
         end
 
         # A result that indicates a security rule match
@@ -33,6 +37,10 @@ module Datadog
           def match?
             true
           end
+
+          def error?
+            false
+          end
         end
 
         # A result that indicates a successful security rules check without a match
@@ -40,6 +48,10 @@ module Datadog
           def match?
             false
           end
+
+          def error?
+            false
+          end
         end
 
         # A result that indicates an internal security library error
@@ -60,6 +72,10 @@ module Datadog
           def match?
             false
           end
+
+          def error?
+            true
+          end
         end
       end
     end

data/lib/datadog/appsec/security_engine/runner.rb

@@ -9,9 +9,13 @@ module Datadog
       class Runner
         SUCCESSFUL_EXECUTION_CODES = [:ok, :match].freeze
 
-        def initialize(waf_context)
+        attr_reader :ruleset_version
+
+        def initialize(handle_ref, ruleset_version:)
           @mutex = Mutex.new
-          @waf_context = waf_context
+          @handle_ref = handle_ref
+          @waf_handle = handle_ref.acquire
+          @ruleset_version = ruleset_version
 
           @debug_tag = "libddwaf:#{WAF::VERSION::STRING} method:ddwaf_run"
         end
@@ -54,14 +58,24 @@ module Datadog
           @mutex.unlock
         end
 
+        def waf_context
+          @waf_context ||= @waf_handle.build_context
+        end
+
+        def waf_addresses
+          @waf_handle.known_addresses
+        end
+
         def finalize!
-          @waf_context.finalize!
+          @waf_context&.finalize!
+        ensure
+          @handle_ref.release(@waf_handle)
         end
 
         private
 
         def try_run(persistent_data, ephemeral_data, timeout)
-          @waf_context.run(persistent_data, ephemeral_data, timeout)
+          waf_context.run(persistent_data, ephemeral_data, timeout)
         rescue WAF::LibDDWAFError => e
           Datadog.logger.debug { "#{@debug_tag} execution error: #{e} backtrace: #{e.backtrace&.first(3)}" }
           AppSec.telemetry.report(e, description: 'libddwaf-rb internal low-level error')

data/lib/datadog/appsec/thread_safe_ref.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # This class is used for referencing an object that might be marked
+    # for finalization in another thread.
+    #
+    # References to the object are counted, and objects marked for finalization
+    # can be safely finalized when their reference count reaches zero.
+    class ThreadSafeRef
+      def initialize(initial_obj, finalizer: :finalize!)
+        @current = initial_obj
+        @finalizer = finalizer
+
+        @counters = Hash.new(0)
+        @outdated = []
+        @mutex = Mutex.new
+      end
+
+      def acquire
+        @mutex.synchronize do
+          @counters[@current] += 1
+
+          @current
+        end
+      end
+
+      def release(obj)
+        @mutex.synchronize do
+          @counters[obj] -= 1
+
+          @outdated.reject! do |outdated_obj|
+            next unless @counters[outdated_obj].zero?
+
+            finalize(outdated_obj)
+          end
+        end
+      end
+
+      def current=(obj)
+        @mutex.synchronize do
+          @outdated << @current
+
+          @current = obj
+        end
+      end
+
+      private
+
+      def finalize(obj)
+        obj.public_send(@finalizer)
+
+        true
+      rescue => e
+        Datadog.logger.debug("Couldn't finalize #{obj.class.name} object, error: #{e.inspect}")
+
+        true
+      end
+    end
+  end
+end

data/lib/datadog/appsec/trace_keeper.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # This class is used to mark trace as manual keep and tag it as ASM product.
+    module TraceKeeper
+      def self.keep!(trace)
+        return unless trace
+
+        # NOTE: This action will not set correct decision maker value, so the
+        #       trace keeping must be done with additional steps below
+        trace.keep!
+
+        # Propagate to downstream services the information that
+        # the current distributed trace is containing at least one ASM event.
+        trace.set_tag(
+          Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
+          Tracing::Sampling::Ext::Decision::ASM
+        )
+        trace.set_distributed_source(Ext::PRODUCT_BIT)
+      end
+    end
+  end
+end

data/lib/datadog/appsec/utils/hash_coercion.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Utils
+      # A module for coercing arbitrary objects into hashes.
+      module HashCoercion
+        # A best effort to coerce an object to a hash with methods known to various
+        # frameworks with a fallback to standard library.
+        #
+        # @param object [Object] The object to coerce.
+        # @return [Hash, nil] The coerced `Hash` or `nil` if the object is not coercible.
+        def self.coerce(object)
+          return object.as_json if object.respond_to?(:as_json)
+          return object.to_hash if object.respond_to?(:to_hash)
+          return object.to_h if object.respond_to?(:to_h)
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec.rb

@@ -34,13 +34,6 @@ module Datadog
         components.appsec&.reconfigure!
       end
 
-      def reconfigure_lock(&block)
-        appsec_component = components.appsec
-        return unless appsec_component
-
-        appsec_component.reconfigure_lock(&block)
-      end
-
       def perform_api_security_check?
         Datadog.configuration.appsec.api_security.enabled &&
           Datadog.configuration.appsec.api_security.sample_rate.sample?

data/lib/datadog/auto_instrument_base.rb

@@ -3,6 +3,7 @@
 module Datadog
   # base methods stubbed for adding auto instrument extensions
   module AutoInstrumentBase
-    def add_auto_instrument; end
+    def add_auto_instrument
+    end
   end
 end

data/lib/datadog/core/configuration/option.rb

@@ -22,9 +22,18 @@ module Datadog
           # Represents an Option precedence level.
           # Each precedence has a `numeric` value; higher values means higher precedence.
           # `name` is for inspection purposes only.
-          Value = Struct.new(:numeric, :name, :origin) do
+
+          class Value
             include Comparable
 
+            attr_accessor :numeric, :name, :origin
+
+            def initialize(numeric, name, origin)
+              @numeric = numeric
+              @name = name
+              @origin = origin
+            end
+
             def <=>(other)
               return nil unless other.is_a?(Value)
 
@@ -172,20 +181,17 @@ module Datadog
         private
 
         def coerce_env_variable(value)
-          return context_exec(value, &@definition.env_parser) if @definition.env_parser
+          env_parser = @definition.env_parser
+          return context_exec(value, &env_parser) if env_parser
 
           case @definition.type
           when :hash
             values = value.split(',') # By default we only want to support comma separated strings
 
-            values.map! do |v|
+            values.each_with_object({}) do |v, hash| # $ Hash[String, String]
               v.gsub!(/\A[\s,]*|[\s,]*\Z/, '')
+              next if v.empty?
 
-              v.empty? ? nil : v
-            end
-
-            values.compact!
-            values.each.with_object({}) do |v, hash|
               pair = v.split(':', 2)
               hash[pair[0]] = pair[1]
             end
@@ -196,14 +202,12 @@ module Datadog
           when :array
             values = value.split(',')
 
-            values.map! do |v|
+            values.each_with_object([]) do |v, arr| # $ Array[String]
               v.gsub!(/\A[\s,]*|[\s,]*\Z/, '')
+              next if v.empty?
 
-              v.empty? ? nil : v
+              arr << v
             end
-
-            values.compact!
-            values
           when :bool
             string_value = value.strip
             string_value = string_value.downcase
@@ -329,18 +333,22 @@ module Datadog
           resolved_env = nil
 
           if definition.env
-            Array(definition.env).each do |env|
-              next if env_vars[env].nil?
+            # @type var env_and_aliases: Array[String]
+            env_and_aliases = Array(definition.env)
+            env_and_aliases.each do |env|
+              env_value = env_vars[env]
+              next if env_value.nil?
 
               resolved_env = env
-              value = coerce_env_variable(env_vars[env])
+              value = coerce_env_variable(env_value)
               break
             end
           end
 
-          if value.nil? && definition.deprecated_env && env_vars[definition.deprecated_env]
+          deprecated_env = definition.deprecated_env ? env_vars[definition.deprecated_env] : nil
+          if value.nil? && deprecated_env
             resolved_env = definition.deprecated_env
-            value = coerce_env_variable(env_vars[definition.deprecated_env])
+            value = coerce_env_variable(deprecated_env)
 
             Datadog::Core.log_deprecation do
               "#{definition.deprecated_env} #{source} is deprecated, use #{definition.env} instead."
@@ -349,9 +357,10 @@ module Datadog
 
           [value, resolved_env]
         rescue ArgumentError
+          env_value = resolved_env ? env_vars[resolved_env] : nil
           raise ArgumentError,
-            "Expected #{source} #{resolved_env} to be a #{@definition.type}, " \
-                              "but '#{env_vars[resolved_env]}' was provided"
+            "Expected #{source} #{resolved_env} to be a #{definition.type}, " \
+            "but '#{env_value}' was provided"
         end
 
         # Anchor object that represents a value that is not set.

data/lib/datadog/core/configuration/option_definition.rb

@@ -22,7 +22,7 @@ module Datadog
           :type,
           :type_options
 
-        def initialize(name, meta = {}, &block)
+        def initialize(name, meta, &block)
           @default = meta[:default]
           @default_proc = meta[:default_proc]
           @env = meta[:env]
@@ -44,7 +44,7 @@ module Datadog
         # Acts as DSL for building OptionDefinitions
         # @public_api
         class Builder
-          class InvalidOptionError < StandardError; end
+          InvalidOptionError = Class.new(StandardError)
 
           attr_reader \
             :helpers

data/lib/datadog/core/configuration/options.rb

@@ -40,14 +40,19 @@ module Datadog
 
           def default_helpers(name)
             option_name = name.to_sym
-
+            # @type var opt_getter: Configuration::OptionDefinition::helper_proc
+            opt_getter = proc do
+              # These Procs uses `get/set_option`, but we only add them to the OptionDefinition helpers here.
+              # Steep is right that these methods are not defined, but we only run these Procs in instance context.
+              get_option(option_name) # steep:ignore NoMethod
+            end
+            # @type var opt_setter: Configuration::OptionDefinition::helper_proc
+            opt_setter = proc do |value|
+              set_option(option_name, value) # steep:ignore NoMethod
+            end
             {
-              option_name.to_sym => proc do
-                get_option(option_name)
-              end,
-              :"#{option_name}=" => proc do |value|
-                set_option(option_name, value)
-              end
+              option_name.to_sym => opt_getter,
+              :"#{option_name}=" => opt_setter
             }
           end
 
@@ -113,6 +118,7 @@ module Datadog
 
             assert_valid_option!(name)
             definition = self.class.options[name]
+            # @type self: Configuration::Options::GenericSettingsClass
             options[name] = definition.build(self)
           end
 

data/lib/datadog/di/boot.rb

@@ -32,3 +32,10 @@ end
 require_relative 'contrib'
 
 Datadog::DI::Contrib.load_now_or_later
+
+if %w[1 true yes].include?(ENV['DD_DYNAMIC_INSTRUMENTATION_ENABLED']) # steep:ignore
+  if ENV['DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE']
+    require_relative 'probe_file_loader'
+    Datadog::DI::ProbeFileLoader.load_now_or_later
+  end
+end

data/lib/datadog/di/component.rb

@@ -112,6 +112,13 @@ module Datadog
         probe_manager.close
         probe_notifier_worker.stop
       end
+
+      def parse_probe_spec_and_notify(probe_spec)
+        probe = ProbeBuilder.build_from_remote_config(probe_spec)
+        payload = probe_notification_builder.build_received(probe)
+        probe_notifier_worker.add_status(payload)
+        probe
+      end
     end
   end
 end