datadog 2.14.0 → 2.16.0

data/lib/datadog/appsec/autoload.rb

@@ -4,7 +4,7 @@ if %w[1 true].include?((ENV['DD_APPSEC_ENABLED'] || '').downcase)
   begin
     require_relative 'contrib/auto_instrument'
     Datadog::AppSec::Contrib::AutoInstrument.patch_all
-  rescue StandardError => e
+  rescue => e
     Kernel.warn(
       '[datadog] AppSec failed to instrument. No security check will be performed. error: ' \
       " #{e.class.name} #{e.message}"

data/lib/datadog/appsec/component.rb

@@ -12,7 +12,25 @@ module Datadog
       class << self
         def build_appsec_component(settings, telemetry:)
           return if !settings.respond_to?(:appsec) || !settings.appsec.enabled
-          return if incompatible_ffi_version?
+
+          ffi_version = Gem.loaded_specs['ffi']&.version
+          unless ffi_version
+            Datadog.logger.warn('FFI gem is not loaded, AppSec will be disabled.')
+            telemetry.error('AppSec: Component not loaded, due to missing FFI gem')
+
+            return
+          end
+
+          if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.3') && ffi_version < Gem::Version.new('1.16.0')
+            Datadog.logger.warn(
+              'AppSec is not supported in Ruby versions above 3.3.0 when using `ffi` versions older than 1.16.0, ' \
+              'and will be forcibly disabled due to a memory leak in `ffi`. ' \
+              'Please upgrade your `ffi` version to 1.16.0 or higher.'
+            )
+            telemetry.error('AppSec: Component not loaded, ffi version is leaky with ruby > 3.3.0')
+
+            return
+          end
 
           processor = create_processor(settings, telemetry)
 
@@ -29,22 +47,6 @@ module Datadog
 
         private
 
-        def incompatible_ffi_version?
-          ffi_version = Gem.loaded_specs['ffi'] && Gem.loaded_specs['ffi'].version
-          return true unless ffi_version
-
-          return false unless Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.3') &&
-            ffi_version < Gem::Version.new('1.16.0')
-
-          Datadog.logger.warn(
-            'AppSec is not supported in Ruby versions above 3.3.0 when using `ffi` versions older than 1.16.0, ' \
-            'and will be forcibly disabled due to a memory leak in `ffi`. ' \
-            'Please upgrade your `ffi` version to 1.16.0 or higher.'
-          )
-
-          true
-        end
-
         def create_processor(settings, telemetry)
           rules = AppSec::Processor::RuleLoader.load_rules(
             telemetry: telemetry,
@@ -59,9 +61,16 @@ module Datadog
 
           exclusions = AppSec::Processor::RuleLoader.load_exclusions(ip_passlist: settings.appsec.ip_passlist)
 
+          # NOTE: This is a temporary solution before the RuleMerger refactoring
+          #       with new RemoteConfig setup
+          processors = rules['processors']
+          scanners = rules['scanners']
+
           ruleset = AppSec::Processor::RuleMerger.merge(
             rules: [rules],
             data: data,
+            scanners: scanners,
+            processors: processors,
             exclusions: exclusions,
             telemetry: telemetry
           )
@@ -86,13 +95,13 @@ module Datadog
         @mutex.synchronize do
           new_processor = Processor.new(ruleset: ruleset, telemetry: telemetry)
 
-          if new_processor && new_processor.ready?
+          if new_processor&.ready?
             old_processor = @processor
 
             @telemetry = telemetry
             @processor = new_processor
 
-            old_processor.finalize if old_processor
+            old_processor&.finalize
           end
         end
       end
@@ -103,7 +112,7 @@ module Datadog
 
       def shutdown!
         @mutex.synchronize do
-          if processor && processor.ready?
+          if processor&.ready?
             processor.finalize
             @processor = nil
           end

data/lib/datadog/appsec/compressed_json.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'zlib'
+require 'stringio'
+
+require_relative '../core/utils/base64'
+
+module Datadog
+  module AppSec
+    # Converts derivative schema payloads into JSON and compresses them into a
+    # base64 encoded string if the payload is worth compressing.
+    #
+    # See: https://github.com/DataDog/dd-trace-rb/pull/3177#issuecomment-1747221082
+    module CompressedJson
+      MIN_SIZE_FOR_COMPRESSION = 260
+
+      def self.dump(payload)
+        value = JSON.dump(payload)
+        return value if value.bytesize < MIN_SIZE_FOR_COMPRESSION
+
+        compress_and_encode(value)
+      rescue ArgumentError, Encoding::UndefinedConversionError, JSON::JSONError => e
+        AppSec.telemetry.report(e, description: 'AppSec: Failed to convert value into JSON')
+
+        nil
+      end
+
+      private_class_method def self.compress_and_encode(payload)
+        Core::Utils::Base64.strict_encode64(
+          Zlib.gzip(payload, level: Zlib::BEST_SPEED, strategy: Zlib::DEFAULT_STRATEGY)
+        )
+      rescue Zlib::Error, TypeError => e
+        AppSec.telemetry.report(e, description: 'AppSec: Failed to compress and encode value')
+
+        nil
+      end
+    end
+  end
+end

data/lib/datadog/appsec/configuration/settings.rb

@@ -131,9 +131,12 @@ module Datadog
                     o.type :string, nilable: true
                     o.setter do |value|
                       if value
-                        raise(ArgumentError, "appsec.templates.html: file not found: #{value}") unless File.exist?(value)
+                        unless File.exist?(value)
+                          raise(ArgumentError,
+                            "appsec.templates.html: file not found: #{value}")
+                        end
 
-                        File.open(value, 'rb', &:read) || ''
+                        File.binread(value) || ''
                       end
                     end
                   end
@@ -143,9 +146,12 @@ module Datadog
                     o.type :string, nilable: true
                     o.setter do |value|
                       if value
-                        raise(ArgumentError, "appsec.templates.json: file not found: #{value}") unless File.exist?(value)
+                        unless File.exist?(value)
+                          raise(ArgumentError,
+                            "appsec.templates.json: file not found: #{value}")
+                        end
 
-                        File.open(value, 'rb', &:read) || ''
+                        File.binread(value) || ''
                       end
                     end
                   end
@@ -155,9 +161,12 @@ module Datadog
                     o.type :string, nilable: true
                     o.setter do |value|
                       if value
-                        raise(ArgumentError, "appsec.templates.text: file not found: #{value}") unless File.exist?(value)
+                        unless File.exist?(value)
+                          raise(ArgumentError,
+                            "appsec.templates.text: file not found: #{value}")
+                        end
 
-                        File.open(value, 'rb', &:read) || ''
+                        File.binread(value) || ''
                       end
                     end
                   end
@@ -237,7 +246,7 @@ module Datadog
 
                     Datadog.logger.warn(
                       'The appsec.auto_user_instrumentation.mode value provided is not supported. ' \
-                      "Supported values are: #{AUTO_USER_INSTRUMENTATION_MODES.join(' | ')}. " \
+                      "Supported values are: #{AUTO_USER_INSTRUMENTATION_MODES.join(" | ")}. " \
                       "Using value: #{DISABLED_AUTO_USER_INSTRUMENTATION_MODE}."
                     )
 
@@ -259,11 +268,13 @@ module Datadog
                       APPSEC_VALID_TRACK_USER_EVENTS_ENABLED_VALUES.include?(env_value.strip.downcase)
                     end
                   end
-                  o.after_set do
-                    Core.log_deprecation(key: :appsec_track_user_events_enabled) do
-                      'The appsec.track_user_events.enabled setting has been deprecated for removal. ' \
-                      'Please remove it from your Datadog.configure block and use ' \
-                      'appsec.auto_user_instrumentation.mode instead.'
+                  o.after_set do |_, _, precedence|
+                    unless precedence == Datadog::Core::Configuration::Option::Precedence::DEFAULT
+                      Core.log_deprecation(key: :appsec_track_user_events_enabled) do
+                        'The appsec.track_user_events.enabled setting is deprecated. ' \
+                        'Please remove it from your Datadog.configure block and use ' \
+                        'appsec.auto_user_instrumentation.mode instead.'
+                      end
                     end
                   end
                 end
@@ -280,18 +291,20 @@ module Datadog
                     else
                       Datadog.logger.warn(
                         'The appsec.track_user_events.mode value provided is not supported.' \
-                        "Supported values are: #{APPSEC_VALID_TRACK_USER_EVENTS_MODE.join(' | ')}." \
+                        "Supported values are: #{APPSEC_VALID_TRACK_USER_EVENTS_MODE.join(" | ")}." \
                         "Using default value: #{SAFE_TRACK_USER_EVENTS_MODE}."
                       )
 
                       SAFE_TRACK_USER_EVENTS_MODE
                     end
                   end
-                  o.after_set do
-                    Core.log_deprecation(key: :appsec_track_user_events_mode) do
-                      'The appsec.track_user_events.mode setting has been deprecated for removal. ' \
-                      'Please remove it from your Datadog.configure block and use ' \
-                      'appsec.auto_user_instrumentation.mode instead.'
+                  o.after_set do |_, _, precedence|
+                    unless precedence == Datadog::Core::Configuration::Option::Precedence::DEFAULT
+                      Core.log_deprecation(key: :appsec_track_user_events_mode) do
+                        'The appsec.track_user_events.mode setting is deprecated. ' \
+                        'Please remove it from your Datadog.configure block and use ' \
+                        'appsec.auto_user_instrumentation.mode instead.'
+                      end
                     end
                   end
                 end

data/lib/datadog/appsec/context.rb

@@ -56,7 +56,7 @@ module Datadog
       end
 
       def extract_schema
-        @waf_runner.run({ 'waf.context.processor' => { 'extract-schema' => true } }, {})
+        @waf_runner.run({'waf.context.processor' => {'extract-schema' => true}}, {})
       end
 
       def export_metrics

data/lib/datadog/appsec/contrib/active_record/instrumentation.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require_relative '../../event'
+require_relative '../../security_event'
+
 module Datadog
   module AppSec
     module Contrib
@@ -28,18 +31,13 @@ module Datadog
             result = context.run_rasp(Ext::RASP_SQLI, {}, ephemeral_data, waf_timeout)
 
             if result.match?
-              Datadog::AppSec::Event.tag_and_keep!(context, result)
-
-              event = {
-                waf_result: result,
-                trace: context.trace,
-                span: context.span,
-                sql: sql,
-                actions: result.actions
-              }
-              context.events << event
-
-              ActionsHandler.handle(result.actions)
+              AppSec::Event.tag_and_keep!(context, result)
+
+              context.events.push(
+                AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+              )
+
+              AppSec::ActionsHandler.handle(result.actions)
             end
           end
 

data/lib/datadog/appsec/contrib/active_record/integration.rb

@@ -13,10 +13,10 @@ module Datadog
 
           MINIMUM_VERSION = Gem::Version.new('4')
 
-          register_as :active_record, auto_patch: false
+          register_as :active_record, auto_patch: true
 
           def self.version
-            Gem.loaded_specs['activerecord'] && Gem.loaded_specs['activerecord'].version
+            Gem.loaded_specs['activerecord']&.version
           end
 
           def self.loaded?

data/lib/datadog/appsec/contrib/active_record/patcher.rb

@@ -53,43 +53,43 @@ module Datadog
 
           def patch_sqlite3_adapter
             instrumentation_module = if ::ActiveRecord.gem_version >= Gem::Version.new('7.1')
-                                       Instrumentation::InternalExecQueryAdapterPatch
-                                     elsif ::ActiveRecord.gem_version.segments.first == 4
-                                       Instrumentation::Rails4ExecQueryAdapterPatch
-                                     else
-                                       Instrumentation::ExecQueryAdapterPatch
-                                     end
+              Instrumentation::InternalExecQueryAdapterPatch
+            elsif ::ActiveRecord.gem_version.segments.first == 4
+              Instrumentation::Rails4ExecQueryAdapterPatch
+            else
+              Instrumentation::ExecQueryAdapterPatch
+            end
 
             ::ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend(instrumentation_module)
           end
 
           def patch_mysql2_adapter
             instrumentation_module = if ::ActiveRecord.gem_version >= Gem::Version.new('7.1')
-                                       Instrumentation::InternalExecQueryAdapterPatch
-                                     elsif ::ActiveRecord.gem_version.segments.first == 4
-                                       Instrumentation::Rails4ExecQueryAdapterPatch
-                                     else
-                                       Instrumentation::ExecQueryAdapterPatch
-                                     end
+              Instrumentation::InternalExecQueryAdapterPatch
+            elsif ::ActiveRecord.gem_version.segments.first == 4
+              Instrumentation::Rails4ExecQueryAdapterPatch
+            else
+              Instrumentation::ExecQueryAdapterPatch
+            end
 
             ::ActiveRecord::ConnectionAdapters::Mysql2Adapter.prepend(instrumentation_module)
           end
 
           def patch_postgresql_adapter
             instrumentation_module = if ::ActiveRecord.gem_version.segments.first == 4
-                                       Instrumentation::Rails4ExecuteAndClearAdapterPatch
-                                     else
-                                       Instrumentation::ExecuteAndClearAdapterPatch
-                                     end
+              Instrumentation::Rails4ExecuteAndClearAdapterPatch
+            else
+              Instrumentation::ExecuteAndClearAdapterPatch
+            end
 
             if defined?(::ActiveRecord::ConnectionAdapters::JdbcAdapter)
               instrumentation_module = if ::ActiveRecord.gem_version >= Gem::Version.new('7.1')
-                                         Instrumentation::InternalExecQueryAdapterPatch
-                                       elsif ::ActiveRecord.gem_version.segments.first == 4
-                                         Instrumentation::Rails4ExecQueryAdapterPatch
-                                       else
-                                         Instrumentation::ExecQueryAdapterPatch
-                                       end
+                Instrumentation::InternalExecQueryAdapterPatch
+              elsif ::ActiveRecord.gem_version.segments.first == 4
+                Instrumentation::Rails4ExecQueryAdapterPatch
+              else
+                Instrumentation::ExecQueryAdapterPatch
+              end
             end
 
             ::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(instrumentation_module)

data/lib/datadog/appsec/contrib/devise/data_extractor.rb

@@ -57,9 +57,8 @@ module Datadog
           def find_devise_scope(object)
             return if ::Devise.mappings.count == 1
 
-            @devise_scopes[object.class.name] ||= begin
-              ::Devise.mappings.each_value.find { |mapping| mapping.class_name == object.class.name }&.name
-            end
+            @devise_scopes[object.class.name] ||= ::Devise.mappings
+              .each_value.find { |mapping| mapping.class_name == object.class.name }&.name
           end
 
           def transform(value)

data/lib/datadog/appsec/contrib/devise/ext.rb

@@ -18,6 +18,7 @@ module Datadog
           TAG_DD_LOGIN_FAILURE_MODE = '_dd.appsec.events.users.login.failure.auto.mode'
 
           TAG_USR_ID = 'usr.id'
+          TAG_SESSION_ID = 'usr.session_id'
           TAG_SIGNUP_TRACK = 'appsec.events.users.signup.track'
           TAG_SIGNUP_USR_ID = 'appsec.events.users.signup.usr.id'
           TAG_SIGNUP_USR_LOGIN = 'appsec.events.users.signup.usr.login'

data/lib/datadog/appsec/contrib/devise/integration.rb

@@ -16,7 +16,7 @@ module Datadog
           register_as :devise, auto_patch: true
 
           def self.version
-            Gem.loaded_specs['devise'] && Gem.loaded_specs['devise'].version
+            Gem.loaded_specs['devise']&.version
           end
 
           def self.loaded?

data/lib/datadog/appsec/contrib/devise/patcher.rb

@@ -30,11 +30,9 @@ module Datadog
           def patch
             ::ActiveSupport.on_load(:before_initialize) do |app|
               GUARD_ONCE_PER_APP[app].run do
-                begin
-                  app.middleware.insert_after(Warden::Manager, TrackingMiddleware)
-                rescue RuntimeError
-                  AppSec.telemetry.error('AppSec: unable to insert Devise TrackingMiddleware')
-                end
+                app.middleware.insert_after(Warden::Manager, TrackingMiddleware)
+              rescue RuntimeError
+                AppSec.telemetry.error('AppSec: unable to insert Devise TrackingMiddleware')
               end
             end
 

data/lib/datadog/appsec/contrib/devise/tracking_middleware.rb

@@ -10,6 +10,7 @@ module Datadog
         # A Rack middleware capable of tracking currently signed user
         class TrackingMiddleware
           WARDEN_KEY = 'warden'
+          SESSION_ID_KEY = 'session_id'
 
           def initialize(app)
             @app = app
@@ -32,16 +33,28 @@ module Datadog
               return @app.call(env)
             end
 
+            # NOTE: Rails session id will be set for unauthenticated users as well,
+            #       so we need to make sure we are tracking only authenticated users.
             id = transform(extract_id(env[WARDEN_KEY]))
+            session_id = env[WARDEN_KEY].raw_session[SESSION_ID_KEY] if id
+
             if id
-              unless context.span.has_tag?(Ext::TAG_USR_ID)
-                context.span[Ext::TAG_USR_ID] = id
+              # NOTE: There is no option to set session id without setting user id via SDK.
+              unless context.span.has_tag?(Ext::TAG_USR_ID) && context.span.has_tag?(Ext::TAG_SESSION_ID)
+                user_id = context.span[Ext::TAG_USR_ID] || id
+                user_session_id = context.span[Ext::TAG_SESSION_ID] || session_id
+
+                # FIXME: The current implementation of event arguments is forsing us
+                #        to bloat User class, and pass nil-value instead of skip
+                #        passing them at first place.
+                #        This is a temporary situation until we refactor events model.
                 AppSec::Instrumentation.gateway.push(
-                  'identity.set_user', AppSec::Instrumentation::Gateway::User.new(id, nil)
+                  'identity.set_user', AppSec::Instrumentation::Gateway::User.new(user_id, nil, user_session_id)
                 )
               end
 
-              context.span[Ext::TAG_DD_USR_ID] = id.to_s
+              context.span[Ext::TAG_USR_ID] ||= id
+              context.span[Ext::TAG_DD_USR_ID] = id
               context.span[Ext::TAG_DD_COLLECTION_MODE] ||= Configuration.auto_user_instrumentation_mode
             end
 

data/lib/datadog/appsec/contrib/excon/integration.rb

@@ -16,7 +16,7 @@ module Datadog
           register_as :excon
 
           def self.version
-            Gem.loaded_specs['excon'] && Gem.loaded_specs['excon'].version
+            Gem.loaded_specs['excon']&.version
           end
 
           def self.loaded?

data/lib/datadog/appsec/contrib/excon/ssrf_detection_middleware.rb

@@ -3,6 +3,9 @@
 
 require 'excon'
 
+require_relative '../../event'
+require_relative '../../security_event'
+
 module Datadog
   module AppSec
     module Contrib
@@ -15,22 +18,18 @@ module Datadog
             context = AppSec.active_context
 
             request_url = URI.join("#{data[:scheme]}://#{data[:host]}", data[:path]).to_s
-            ephemeral_data = { 'server.io.net.url' => request_url }
+            ephemeral_data = {'server.io.net.url' => request_url}
 
             result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, Datadog.configuration.appsec.waf_timeout)
 
             if result.match?
-              Datadog::AppSec::Event.tag_and_keep!(context, result)
+              AppSec::Event.tag_and_keep!(context, result)
 
-              context.events << {
-                waf_result: result,
-                trace: context.trace,
-                span: context.span,
-                request_url: request_url,
-                actions: result.actions
-              }
+              context.events.push(
+                AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+              )
 
-              ActionsHandler.handle(result.actions)
+              AppSec::ActionsHandler.handle(result.actions)
             end
 
             super

data/lib/datadog/appsec/contrib/faraday/integration.rb

@@ -17,7 +17,7 @@ module Datadog
           register_as :faraday, auto_patch: true
 
           def self.version
-            Gem.loaded_specs['faraday'] && Gem.loaded_specs['faraday'].version
+            Gem.loaded_specs['faraday']&.version
           end
 
           def self.loaded?

data/lib/datadog/appsec/contrib/faraday/ssrf_detection_middleware.rb

@@ -1,6 +1,9 @@
 # rubocop:disable Naming/FileName
 # frozen_string_literal: true
 
+require_relative '../../event'
+require_relative '../../security_event'
+
 module Datadog
   module AppSec
     module Contrib
@@ -19,17 +22,13 @@ module Datadog
             result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, Datadog.configuration.appsec.waf_timeout)
 
             if result.match?
-              Datadog::AppSec::Event.tag_and_keep!(context, result)
+              AppSec::Event.tag_and_keep!(context, result)
 
-              context.events << {
-                waf_result: result,
-                trace: context.trace,
-                span: context.span,
-                request_url: request_env.url,
-                actions: result.actions
-              }
+              context.events.push(
+                AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+              )
 
-              ActionsHandler.handle(result.actions)
+              AppSec::ActionsHandler.handle(result.actions)
             end
 
             @app.call(request_env)

data/lib/datadog/appsec/contrib/graphql/gateway/watcher.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 require 'json'
+
+require_relative '../../../event'
+require_relative '../../../security_event'
 require_relative '../../../instrumentation/gateway'
 
 module Datadog
@@ -29,17 +32,13 @@ module Datadog
                     result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
                     if result.match?
-                      Datadog::AppSec::Event.tag_and_keep!(context, result)
+                      AppSec::Event.tag_and_keep!(context, result)
 
-                      context.events << {
-                        waf_result: result,
-                        trace: context.trace,
-                        span: context.span,
-                        multiplex: gateway_multiplex,
-                        actions: result.actions
-                      }
+                      context.events.push(
+                        AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+                      )
 
-                      Datadog::AppSec::ActionsHandler.handle(result.actions)
+                      AppSec::ActionsHandler.handle(result.actions)
                     end
                   end
 

data/lib/datadog/appsec/contrib/graphql/integration.rb

@@ -23,7 +23,7 @@ module Datadog
           register_as :graphql, auto_patch: false
 
           def self.version
-            Gem.loaded_specs['graphql'] && Gem.loaded_specs['graphql'].version
+            Gem.loaded_specs['graphql']&.version
           end
 
           def self.loaded?