datadog 2.12.1 → 2.19.0

data/lib/datadog/appsec/autoload.rb

@@ -4,7 +4,7 @@ if %w[1 true].include?((ENV['DD_APPSEC_ENABLED'] || '').downcase)
   begin
     require_relative 'contrib/auto_instrument'
     Datadog::AppSec::Contrib::AutoInstrument.patch_all
-  rescue StandardError => e
+  rescue => e
     Kernel.warn(
       '[datadog] AppSec failed to instrument. No security check will be performed. error: ' \
       " #{e.class.name} #{e.message}"

data/lib/datadog/appsec/component.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require_relative 'processor'
-require_relative 'processor/rule_merger'
+require_relative 'security_engine/engine'
+require_relative 'security_engine/runner'
 require_relative 'processor/rule_loader'
 require_relative 'actions_handler'
 
@@ -12,9 +12,28 @@ module Datadog
       class << self
         def build_appsec_component(settings, telemetry:)
           return if !settings.respond_to?(:appsec) || !settings.appsec.enabled
-          return if incompatible_ffi_version?
 
-          processor = create_processor(settings, telemetry)
+          ffi_version = Gem.loaded_specs['ffi']&.version
+          unless ffi_version
+            Datadog.logger.warn('FFI gem is not loaded, AppSec will be disabled.')
+            telemetry.error('AppSec: Component not loaded, due to missing FFI gem')
+
+            return
+          end
+
+          if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.3') && ffi_version < Gem::Version.new('1.16.0')
+            Datadog.logger.warn(
+              'AppSec is not supported in Ruby versions above 3.3.0 when using `ffi` versions older than 1.16.0, ' \
+              'and will be forcibly disabled due to a memory leak in `ffi`. ' \
+              'Please upgrade your `ffi` version to 1.16.0 or higher.'
+            )
+            telemetry.error('AppSec: Component not loaded, ffi version is leaky with ruby > 3.3.0')
+
+            return
+          end
+
+          require_libddwaf(telemetry: telemetry)
+          Datadog::AppSec::WAF.logger = Datadog.logger if Datadog.logger.debug? && settings.appsec.waf_debug
 
           # We want to always instrument user events when AppSec is enabled.
           # There could be cases in which users use the DD_APPSEC_ENABLED Env variable to
@@ -24,76 +43,44 @@ module Datadog
           devise_integration = Datadog::AppSec::Contrib::Devise::Integration.new
           settings.appsec.instrument(:devise) unless devise_integration.patcher.patched?
 
-          new(processor, telemetry)
-        end
-
-        private
-
-        def incompatible_ffi_version?
-          ffi_version = Gem.loaded_specs['ffi'] && Gem.loaded_specs['ffi'].version
-          return true unless ffi_version
+          security_engine = SecurityEngine::Engine.new(appsec_settings: settings.appsec, telemetry: telemetry)
+          new(security_engine: security_engine, telemetry: telemetry)
+        rescue
+          Datadog.logger.warn('AppSec is disabled, see logged errors above')
 
-          return false unless Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.3') &&
-            ffi_version < Gem::Version.new('1.16.0')
-
-          Datadog.logger.warn(
-            'AppSec is not supported in Ruby versions above 3.3.0 when using `ffi` versions older than 1.16.0, ' \
-            'and will be forcibly disabled due to a memory leak in `ffi`. ' \
-            'Please upgrade your `ffi` version to 1.16.0 or higher.'
-          )
-
-          true
+          nil
         end
 
-        def create_processor(settings, telemetry)
-          rules = AppSec::Processor::RuleLoader.load_rules(
-            telemetry: telemetry,
-            ruleset: settings.appsec.ruleset
-          )
-          return nil unless rules
-
-          data = AppSec::Processor::RuleLoader.load_data(
-            ip_denylist: settings.appsec.ip_denylist,
-            user_id_denylist: settings.appsec.user_id_denylist,
-          )
+        private
 
-          exclusions = AppSec::Processor::RuleLoader.load_exclusions(ip_passlist: settings.appsec.ip_passlist)
+        def require_libddwaf(telemetry:)
+          require('libddwaf')
+        rescue LoadError => e
+          libddwaf_platform = Gem.loaded_specs['libddwaf']&.platform || 'unknown'
+          ruby_platforms = Gem.platforms.map(&:to_s)
 
-          ruleset = AppSec::Processor::RuleMerger.merge(
-            rules: [rules],
-            data: data,
-            exclusions: exclusions,
-            telemetry: telemetry
-          )
+          error_message = "libddwaf failed to load - installed platform: #{libddwaf_platform}, " \
+            "ruby platforms: #{ruby_platforms}"
 
-          processor = Processor.new(ruleset: ruleset, telemetry: telemetry)
-          return nil unless processor.ready?
+          Datadog.logger.error("#{error_message}, error #{e.inspect}")
+          telemetry.report(e, description: error_message)
 
-          processor
+          raise e
         end
       end
 
-      attr_reader :processor, :telemetry
+      attr_reader :security_engine, :telemetry
 
-      def initialize(processor, telemetry)
-        @processor = processor
+      def initialize(security_engine:, telemetry:)
+        @security_engine = security_engine
         @telemetry = telemetry
 
         @mutex = Mutex.new
       end
 
-      def reconfigure(ruleset:, telemetry:)
+      def reconfigure!
         @mutex.synchronize do
-          new_processor = Processor.new(ruleset: ruleset, telemetry: telemetry)
-
-          if new_processor && new_processor.ready?
-            old_processor = @processor
-
-            @telemetry = telemetry
-            @processor = new_processor
-
-            old_processor.finalize if old_processor
-          end
+          security_engine.reconfigure!
         end
       end
 
@@ -103,10 +90,8 @@ module Datadog
 
       def shutdown!
         @mutex.synchronize do
-          if processor && processor.ready?
-            processor.finalize
-            @processor = nil
-          end
+          security_engine.finalize!
+          @security_engine = nil
         end
       end
     end

data/lib/datadog/appsec/compressed_json.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'zlib'
+require 'stringio'
+
+require_relative '../core/utils/base64'
+
+module Datadog
+  module AppSec
+    # Converts derivative schema payloads into JSON and compresses them into a
+    # base64 encoded string if the payload is worth compressing.
+    #
+    # See: https://github.com/DataDog/dd-trace-rb/pull/3177#issuecomment-1747221082
+    module CompressedJson
+      MIN_SIZE_FOR_COMPRESSION = 260
+
+      def self.dump(payload)
+        value = JSON.dump(payload)
+        return value if value.bytesize < MIN_SIZE_FOR_COMPRESSION
+
+        compress_and_encode(value)
+      rescue ArgumentError, Encoding::UndefinedConversionError, JSON::JSONError => e
+        AppSec.telemetry.report(e, description: 'AppSec: Failed to convert value into JSON')
+
+        nil
+      end
+
+      private_class_method def self.compress_and_encode(payload)
+        Core::Utils::Base64.strict_encode64(
+          Zlib.gzip(payload, level: Zlib::BEST_SPEED, strategy: Zlib::DEFAULT_STRATEGY)
+        )
+      rescue Zlib::Error, TypeError => e
+        AppSec.telemetry.report(e, description: 'AppSec: Failed to compress and encode value')
+
+        nil
+      end
+    end
+  end
+end

data/lib/datadog/appsec/configuration/settings.rb

@@ -80,16 +80,49 @@ module Datadog
 
               option :ip_passlist do |o|
                 o.default []
+
+                o.setter do |value|
+                  next value if value.nil? || value.empty?
+
+                  Datadog::Core.log_deprecation(disallowed_next_major: false) do
+                    'The ip_passlist setting is deprecated and will be removed in the next release. ' \
+                    'Please migrate this configuration to your service settings via the Datadog UI'
+                  end
+
+                  value
+                end
               end
 
               option :ip_denylist do |o|
                 o.type :array
                 o.default []
+
+                o.setter do |value|
+                  next value if value.nil? || value.empty?
+
+                  Datadog::Core.log_deprecation(disallowed_next_major: false) do
+                    'The ip_denylist setting is deprecated and will be removed in the next release. ' \
+                    'Please migrate this configuration to your service settings via the Datadog UI'
+                  end
+
+                  value
+                end
               end
 
               option :user_id_denylist do |o|
                 o.type :array
                 o.default []
+
+                o.setter do |value|
+                  next value if value.nil? || value.empty?
+
+                  Datadog::Core.log_deprecation(disallowed_next_major: false) do
+                    'The user_id_denylist setting is deprecated and will be removed in the next release. ' \
+                    'Please migrate this configuration to your service settings via the Datadog UI'
+                  end
+
+                  value
+                end
               end
 
               option :waf_timeout do |o|
@@ -131,9 +164,12 @@ module Datadog
                     o.type :string, nilable: true
                     o.setter do |value|
                       if value
-                        raise(ArgumentError, "appsec.templates.html: file not found: #{value}") unless File.exist?(value)
+                        unless File.exist?(value)
+                          raise(ArgumentError,
+                            "appsec.templates.html: file not found: #{value}")
+                        end
 
-                        File.open(value, 'rb', &:read) || ''
+                        File.binread(value) || ''
                       end
                     end
                   end
@@ -143,9 +179,12 @@ module Datadog
                     o.type :string, nilable: true
                     o.setter do |value|
                       if value
-                        raise(ArgumentError, "appsec.templates.json: file not found: #{value}") unless File.exist?(value)
+                        unless File.exist?(value)
+                          raise(ArgumentError,
+                            "appsec.templates.json: file not found: #{value}")
+                        end
 
-                        File.open(value, 'rb', &:read) || ''
+                        File.binread(value) || ''
                       end
                     end
                   end
@@ -155,15 +194,78 @@ module Datadog
                     o.type :string, nilable: true
                     o.setter do |value|
                       if value
-                        raise(ArgumentError, "appsec.templates.text: file not found: #{value}") unless File.exist?(value)
+                        unless File.exist?(value)
+                          raise(ArgumentError,
+                            "appsec.templates.text: file not found: #{value}")
+                        end
 
-                        File.open(value, 'rb', &:read) || ''
+                        File.binread(value) || ''
                       end
                     end
                   end
                 end
               end
 
+              settings :stack_trace do
+                option :enabled do |o|
+                  o.type :bool
+                  o.env 'DD_APPSEC_STACK_TRACE_ENABLED'
+                  o.default true
+                end
+
+                # The maximum number of stack trace frames to collect for each stack trace.
+                #
+                # If the stack trace exceeds this limit, the frames are dropped from the middle of the stack trace:
+                # 75% of the frames are kept from the top of the stack trace and 25% from the bottom
+                # (this percentage is also configurable).
+                #
+                # Minimum value is 10.
+                # Set to zero if you don't want any frames to be dropped.
+                #
+                # Default value is 32
+                option :max_depth do |o|
+                  o.type :int
+                  o.env 'DD_APPSEC_MAX_STACK_TRACE_DEPTH'
+                  o.default 32
+
+                  o.setter do |value|
+                    value = 0 if value < 0
+                    value
+                  end
+                end
+
+                # The percentage of frames to keep from the top of the stack trace.
+                #
+                # Default value is 75
+                option :top_percentage do |o|
+                  o.type :int
+                  o.env 'DD_APPSEC_MAX_STACK_TRACE_DEPTH_TOP_PERCENT'
+                  o.default 75
+
+                  o.setter do |value|
+                    value = 100 if value > 100
+                    value = 0 if value.negative?
+                    value
+                  end
+                end
+
+                # Maximum number of stack traces to collect per span.
+                #
+                # Set to zero if you want to collect all stack traces.
+                #
+                # Default value is 2
+                option :max_stack_traces do |o|
+                  o.type :int
+                  o.env 'DD_APPSEC_MAX_STACK_TRACES'
+                  o.default 2
+
+                  o.setter do |value|
+                    value = 0 if value < 0
+                    value
+                  end
+                end
+              end
+
               settings :auto_user_instrumentation do
                 define_method(:enabled?) { get_option(:mode) != DISABLED_AUTO_USER_INSTRUMENTATION_MODE }
 
@@ -177,11 +279,11 @@ module Datadog
 
                     Datadog.logger.warn(
                       'The appsec.auto_user_instrumentation.mode value provided is not supported. ' \
-                      "Supported values are: #{AUTO_USER_INSTRUMENTATION_MODES.join(' | ')}. " \
-                      "Using default value: #{IDENTIFICATION_AUTO_USER_INSTRUMENTATION_MODE}."
+                      "Supported values are: #{AUTO_USER_INSTRUMENTATION_MODES.join(" | ")}. " \
+                      "Using value: #{DISABLED_AUTO_USER_INSTRUMENTATION_MODE}."
                     )
 
-                    IDENTIFICATION_AUTO_USER_INSTRUMENTATION_MODE
+                    DISABLED_AUTO_USER_INSTRUMENTATION_MODE
                   end
                 end
               end
@@ -199,11 +301,13 @@ module Datadog
                       APPSEC_VALID_TRACK_USER_EVENTS_ENABLED_VALUES.include?(env_value.strip.downcase)
                     end
                   end
-                  o.after_set do
-                    Core.log_deprecation(key: :appsec_track_user_events_enabled) do
-                      'The appsec.track_user_events.enabled setting has been deprecated for removal. ' \
-                      'Please remove it from your Datadog.configure block and use ' \
-                      'appsec.auto_user_instrumentation.mode instead.'
+                  o.after_set do |_, _, precedence|
+                    unless precedence == Datadog::Core::Configuration::Option::Precedence::DEFAULT
+                      Core.log_deprecation(key: :appsec_track_user_events_enabled) do
+                        'The appsec.track_user_events.enabled setting is deprecated. ' \
+                        'Please remove it from your Datadog.configure block and use ' \
+                        'appsec.auto_user_instrumentation.mode instead.'
+                      end
                     end
                   end
                 end
@@ -220,30 +324,48 @@ module Datadog
                     else
                       Datadog.logger.warn(
                         'The appsec.track_user_events.mode value provided is not supported.' \
-                        "Supported values are: #{APPSEC_VALID_TRACK_USER_EVENTS_MODE.join(' | ')}." \
+                        "Supported values are: #{APPSEC_VALID_TRACK_USER_EVENTS_MODE.join(" | ")}." \
                         "Using default value: #{SAFE_TRACK_USER_EVENTS_MODE}."
                       )
 
                       SAFE_TRACK_USER_EVENTS_MODE
                     end
                   end
-                  o.after_set do
-                    Core.log_deprecation(key: :appsec_track_user_events_mode) do
-                      'The appsec.track_user_events.mode setting has been deprecated for removal. ' \
-                      'Please remove it from your Datadog.configure block and use ' \
-                      'appsec.auto_user_instrumentation.mode instead.'
+                  o.after_set do |_, _, precedence|
+                    unless precedence == Datadog::Core::Configuration::Option::Precedence::DEFAULT
+                      Core.log_deprecation(key: :appsec_track_user_events_mode) do
+                        'The appsec.track_user_events.mode setting is deprecated. ' \
+                        'Please remove it from your Datadog.configure block and use ' \
+                        'appsec.auto_user_instrumentation.mode instead.'
+                      end
                     end
                   end
                 end
               end
 
               settings :api_security do
+                define_method(:enabled?) { get_option(:enabled) }
+
                 option :enabled do |o|
                   o.type :bool
-                  o.env 'DD_EXPERIMENTAL_API_SECURITY_ENABLED'
-                  o.default false
+                  o.env 'DD_API_SECURITY_ENABLED'
+                  o.default true
                 end
 
+                # NOTE: Unfortunately, we have to go with Float due to other libs
+                #       setup, even tho we don't plan to support sub-second delays.
+                #
+                # WARNING: The value will be converted to Integer.
+                option :sample_delay do |o|
+                  o.type :float
+                  o.env 'DD_API_SECURITY_SAMPLE_DELAY'
+                  o.default 30
+                  o.setter do |value|
+                    value.to_i
+                  end
+                end
+
+                # DEV-3.0: Remove `api_security.sample_rate` option
                 option :sample_rate do |o|
                   o.type :float
                   o.env 'DD_API_SECURITY_REQUEST_SAMPLE_RATE'
@@ -252,6 +374,15 @@ module Datadog
                     value = 1 if value > 1
                     SampleRate.new(value)
                   end
+                  o.after_set do |_, _, precedence|
+                    next if precedence == Datadog::Core::Configuration::Option::Precedence::DEFAULT
+
+                    Core.log_deprecation(key: :appsec_api_security_sample_rate) do
+                      'The appsec.api_security.sample_rate setting is deprecated. ' \
+                      'Please remove it from your Datadog.configure block and use ' \
+                      'appsec.api_security.sample_delay instead.'
+                    end
+                  end
                 end
               end
 
@@ -259,14 +390,6 @@ module Datadog
                 o.type :bool, nilable: true
                 o.env 'DD_APPSEC_SCA_ENABLED'
               end
-
-              settings :standalone do
-                option :enabled do |o|
-                  o.type :bool
-                  o.env 'DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED'
-                  o.default false
-                end
-              end
             end
           end
         end

data/lib/datadog/appsec/context.rb

@@ -9,6 +9,7 @@ module Datadog
     class Context
       ActiveContextError = Class.new(StandardError)
 
+      # TODO: add delegators for active trace span
       attr_reader :trace, :span, :events
 
       class << self
@@ -20,7 +21,7 @@ module Datadog
         end
 
         def deactivate
-          active&.finalize
+          active&.finalize!
         ensure
           Thread.current[Ext::ACTIVE_CONTEXT_KEY] = nil
         end
@@ -30,12 +31,11 @@ module Datadog
         end
       end
 
-      def initialize(trace, span, security_engine)
+      def initialize(trace, span, waf_runner)
         @trace = trace
         @span = span
         @events = []
-        @security_engine = security_engine
-        @waf_runner = security_engine.new_runner
+        @waf_runner = waf_runner
         @metrics = Metrics::Collector.new
       end
 
@@ -56,7 +56,7 @@ module Datadog
       end
 
       def extract_schema
-        @waf_runner.run({ 'waf.context.processor' => { 'extract-schema' => true } }, {})
+        @waf_runner.run({'waf.context.processor' => {'extract-schema' => true}}, {})
       end
 
       def export_metrics
@@ -66,8 +66,8 @@ module Datadog
         Metrics::Exporter.export_rasp_metrics(@metrics.rasp, @span)
       end
 
-      def finalize
-        @waf_runner.finalize
+      def finalize!
+        @waf_runner.finalize!
       end
     end
   end

data/lib/datadog/appsec/contrib/active_record/instrumentation.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require_relative '../../event'
+require_relative '../../security_event'
+
 module Datadog
   module AppSec
     module Contrib
@@ -28,18 +31,13 @@ module Datadog
             result = context.run_rasp(Ext::RASP_SQLI, {}, ephemeral_data, waf_timeout)
 
             if result.match?
-              Datadog::AppSec::Event.tag_and_keep!(context, result)
-
-              event = {
-                waf_result: result,
-                trace: context.trace,
-                span: context.span,
-                sql: sql,
-                actions: result.actions
-              }
-              context.events << event
-
-              ActionsHandler.handle(result.actions)
+              AppSec::Event.tag_and_keep!(context, result)
+
+              context.events.push(
+                AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+              )
+
+              AppSec::ActionsHandler.handle(result.actions)
             end
           end
 

data/lib/datadog/appsec/contrib/active_record/integration.rb

@@ -13,10 +13,10 @@ module Datadog
 
           MINIMUM_VERSION = Gem::Version.new('4')
 
-          register_as :active_record, auto_patch: false
+          register_as :active_record, auto_patch: true
 
           def self.version
-            Gem.loaded_specs['activerecord'] && Gem.loaded_specs['activerecord'].version
+            Gem.loaded_specs['activerecord']&.version
           end
 
           def self.loaded?

data/lib/datadog/appsec/contrib/active_record/patcher.rb

@@ -53,43 +53,43 @@ module Datadog
 
           def patch_sqlite3_adapter
             instrumentation_module = if ::ActiveRecord.gem_version >= Gem::Version.new('7.1')
-                                       Instrumentation::InternalExecQueryAdapterPatch
-                                     elsif ::ActiveRecord.gem_version.segments.first == 4
-                                       Instrumentation::Rails4ExecQueryAdapterPatch
-                                     else
-                                       Instrumentation::ExecQueryAdapterPatch
-                                     end
+              Instrumentation::InternalExecQueryAdapterPatch
+            elsif ::ActiveRecord.gem_version.segments.first == 4
+              Instrumentation::Rails4ExecQueryAdapterPatch
+            else
+              Instrumentation::ExecQueryAdapterPatch
+            end
 
             ::ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend(instrumentation_module)
           end
 
           def patch_mysql2_adapter
             instrumentation_module = if ::ActiveRecord.gem_version >= Gem::Version.new('7.1')
-                                       Instrumentation::InternalExecQueryAdapterPatch
-                                     elsif ::ActiveRecord.gem_version.segments.first == 4
-                                       Instrumentation::Rails4ExecQueryAdapterPatch
-                                     else
-                                       Instrumentation::ExecQueryAdapterPatch
-                                     end
+              Instrumentation::InternalExecQueryAdapterPatch
+            elsif ::ActiveRecord.gem_version.segments.first == 4
+              Instrumentation::Rails4ExecQueryAdapterPatch
+            else
+              Instrumentation::ExecQueryAdapterPatch
+            end
 
             ::ActiveRecord::ConnectionAdapters::Mysql2Adapter.prepend(instrumentation_module)
           end
 
           def patch_postgresql_adapter
             instrumentation_module = if ::ActiveRecord.gem_version.segments.first == 4
-                                       Instrumentation::Rails4ExecuteAndClearAdapterPatch
-                                     else
-                                       Instrumentation::ExecuteAndClearAdapterPatch
-                                     end
+              Instrumentation::Rails4ExecuteAndClearAdapterPatch
+            else
+              Instrumentation::ExecuteAndClearAdapterPatch
+            end
 
             if defined?(::ActiveRecord::ConnectionAdapters::JdbcAdapter)
               instrumentation_module = if ::ActiveRecord.gem_version >= Gem::Version.new('7.1')
-                                         Instrumentation::InternalExecQueryAdapterPatch
-                                       elsif ::ActiveRecord.gem_version.segments.first == 4
-                                         Instrumentation::Rails4ExecQueryAdapterPatch
-                                       else
-                                         Instrumentation::ExecQueryAdapterPatch
-                                       end
+                Instrumentation::InternalExecQueryAdapterPatch
+              elsif ::ActiveRecord.gem_version.segments.first == 4
+                Instrumentation::Rails4ExecQueryAdapterPatch
+              else
+                Instrumentation::ExecQueryAdapterPatch
+              end
             end
 
             ::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(instrumentation_module)

data/lib/datadog/appsec/contrib/auto_instrument.rb

@@ -9,7 +9,7 @@ module Datadog
         def self.patch_all
           integrations = []
 
-          Datadog::AppSec::Contrib::Integration.registry.each do |_name, integration|
+          Datadog::AppSec::Contrib::Integration.registry.each_value do |integration|
             next unless integration.klass.auto_instrument?
 
             integrations << integration.name