datadog 2.7.1 → 2.17.0

data/lib/datadog/appsec/configuration/settings.rb

@@ -12,14 +12,29 @@ module Datadog
         DEFAULT_OBFUSCATOR_KEY_REGEX = '(?i)pass|pw(?:or)?d|secret|(?:api|private|public|access)[_-]?key|token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|jsessionid|phpsessid|asp\.net[_-]sessionid|sid|jwt'
         DEFAULT_OBFUSCATOR_VALUE_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:[_-]?phrase)?|secret(?:[_-]?key)?|(?:(?:api|private|public|access)[_-]?)key(?:[_-]?id)?|(?:(?:auth|access|id|refresh)[_-]?)?token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?|jsessionid|phpsessid|asp\.net(?:[_-]|-)sessionid|sid|jwt)(?:\s*=[^;]|"\s*:\s*"[^"]+")|bearer\s+[a-z0-9\._\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\w=-]+\.ey[I-L][\w=-]+(?:\.[\w.+\/=-]+)?|[\-]{5}BEGIN[a-z\s]+PRIVATE\sKEY[\-]{5}[^\-]+[\-]{5}END[a-z\s]+PRIVATE\sKEY|ssh-rsa\s*[a-z0-9\/\.+]{100,}'
         # rubocop:enable Layout/LineLength
+
+        DISABLED_AUTO_USER_INSTRUMENTATION_MODE = 'disabled'
+        ANONYMIZATION_AUTO_USER_INSTRUMENTATION_MODE = 'anonymization'
+        IDENTIFICATION_AUTO_USER_INSTRUMENTATION_MODE = 'identification'
+        AUTO_USER_INSTRUMENTATION_MODES = [
+          DISABLED_AUTO_USER_INSTRUMENTATION_MODE,
+          ANONYMIZATION_AUTO_USER_INSTRUMENTATION_MODE,
+          IDENTIFICATION_AUTO_USER_INSTRUMENTATION_MODE
+        ].freeze
+        AUTO_USER_INSTRUMENTATION_MODES_ALIASES = {
+          'ident' => IDENTIFICATION_AUTO_USER_INSTRUMENTATION_MODE,
+          'anon' => ANONYMIZATION_AUTO_USER_INSTRUMENTATION_MODE,
+        }.freeze
+
+        # NOTE: These two constants are deprecated
+        SAFE_TRACK_USER_EVENTS_MODE = 'safe'
+        EXTENDED_TRACK_USER_EVENTS_MODE = 'extended'
         APPSEC_VALID_TRACK_USER_EVENTS_MODE = [
-          'safe',
-          'extended'
+          SAFE_TRACK_USER_EVENTS_MODE, EXTENDED_TRACK_USER_EVENTS_MODE
         ].freeze
-        APPSEC_VALID_TRACK_USER_EVENTS_ENABLED_VALUES = [
-          '1',
-          'true'
-        ].concat(APPSEC_VALID_TRACK_USER_EVENTS_MODE).freeze
+        APPSEC_VALID_TRACK_USER_EVENTS_ENABLED_VALUES = ['1', 'true'].concat(
+          APPSEC_VALID_TRACK_USER_EVENTS_MODE
+        ).freeze
 
         def self.extended(base)
           base = base.singleton_class unless base.is_a?(Class)
@@ -49,6 +64,15 @@ module Datadog
                 end
               end
 
+              # RASP or Runtime Application Self-Protection
+              # is a collection of techniques and heuristics aimed at detecting malicious inputs and preventing
+              # any potential side-effects on the application resulting from the use of said malicious inputs.
+              option :rasp_enabled do |o|
+                o.type :bool, nilable: true
+                o.env 'DD_APPSEC_RASP_ENABLED'
+                o.default true
+              end
+
               option :ruleset do |o|
                 o.env 'DD_APPSEC_RULES'
                 o.default :recommended
@@ -107,9 +131,12 @@ module Datadog
                     o.type :string, nilable: true
                     o.setter do |value|
                       if value
-                        raise(ArgumentError, "appsec.templates.html: file not found: #{value}") unless File.exist?(value)
+                        unless File.exist?(value)
+                          raise(ArgumentError,
+                            "appsec.templates.html: file not found: #{value}")
+                        end
 
-                        File.open(value, 'rb', &:read) || ''
+                        File.binread(value) || ''
                       end
                     end
                   end
@@ -119,9 +146,12 @@ module Datadog
                     o.type :string, nilable: true
                     o.setter do |value|
                       if value
-                        raise(ArgumentError, "appsec.templates.json: file not found: #{value}") unless File.exist?(value)
+                        unless File.exist?(value)
+                          raise(ArgumentError,
+                            "appsec.templates.json: file not found: #{value}")
+                        end
 
-                        File.open(value, 'rb', &:read) || ''
+                        File.binread(value) || ''
                       end
                     end
                   end
@@ -131,15 +161,101 @@ module Datadog
                     o.type :string, nilable: true
                     o.setter do |value|
                       if value
-                        raise(ArgumentError, "appsec.templates.text: file not found: #{value}") unless File.exist?(value)
+                        unless File.exist?(value)
+                          raise(ArgumentError,
+                            "appsec.templates.text: file not found: #{value}")
+                        end
 
-                        File.open(value, 'rb', &:read) || ''
+                        File.binread(value) || ''
                       end
                     end
                   end
                 end
               end
 
+              settings :stack_trace do
+                option :enabled do |o|
+                  o.type :bool
+                  o.env 'DD_APPSEC_STACK_TRACE_ENABLED'
+                  o.default true
+                end
+
+                # The maximum number of stack trace frames to collect for each stack trace.
+                #
+                # If the stack trace exceeds this limit, the frames are dropped from the middle of the stack trace:
+                # 75% of the frames are kept from the top of the stack trace and 25% from the bottom
+                # (this percentage is also configurable).
+                #
+                # Minimum value is 10.
+                # Set to zero if you don't want any frames to be dropped.
+                #
+                # Default value is 32
+                option :max_depth do |o|
+                  o.type :int
+                  o.env 'DD_APPSEC_MAX_STACK_TRACE_DEPTH'
+                  o.default 32
+
+                  o.setter do |value|
+                    value = 0 if value < 0
+                    value
+                  end
+                end
+
+                # The percentage of frames to keep from the top of the stack trace.
+                #
+                # Default value is 75
+                option :top_percentage do |o|
+                  o.type :int
+                  o.env 'DD_APPSEC_MAX_STACK_TRACE_DEPTH_TOP_PERCENT'
+                  o.default 75
+
+                  o.setter do |value|
+                    value = 100 if value > 100
+                    value = 0 if value.negative?
+                    value
+                  end
+                end
+
+                # Maximum number of stack traces to collect per span.
+                #
+                # Set to zero if you want to collect all stack traces.
+                #
+                # Default value is 2
+                option :max_stack_traces do |o|
+                  o.type :int
+                  o.env 'DD_APPSEC_MAX_STACK_TRACES'
+                  o.default 2
+
+                  o.setter do |value|
+                    value = 0 if value < 0
+                    value
+                  end
+                end
+              end
+
+              settings :auto_user_instrumentation do
+                define_method(:enabled?) { get_option(:mode) != DISABLED_AUTO_USER_INSTRUMENTATION_MODE }
+
+                option :mode do |o|
+                  o.type :string
+                  o.env 'DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE'
+                  o.default IDENTIFICATION_AUTO_USER_INSTRUMENTATION_MODE
+                  o.setter do |value|
+                    mode = AUTO_USER_INSTRUMENTATION_MODES_ALIASES.fetch(value, value)
+                    next mode if AUTO_USER_INSTRUMENTATION_MODES.include?(mode)
+
+                    Datadog.logger.warn(
+                      'The appsec.auto_user_instrumentation.mode value provided is not supported. ' \
+                      "Supported values are: #{AUTO_USER_INSTRUMENTATION_MODES.join(" | ")}. " \
+                      "Using value: #{DISABLED_AUTO_USER_INSTRUMENTATION_MODE}."
+                    )
+
+                    DISABLED_AUTO_USER_INSTRUMENTATION_MODE
+                  end
+                end
+              end
+
+              # DEV-3.0: Remove `track_user_events.enabled` and `track_user_events.mode` options
               settings :track_user_events do
                 option :enabled do |o|
                   o.default true
@@ -152,24 +268,43 @@ module Datadog
                       APPSEC_VALID_TRACK_USER_EVENTS_ENABLED_VALUES.include?(env_value.strip.downcase)
                     end
                   end
+                  o.after_set do |_, _, precedence|
+                    unless precedence == Datadog::Core::Configuration::Option::Precedence::DEFAULT
+                      Core.log_deprecation(key: :appsec_track_user_events_enabled) do
+                        'The appsec.track_user_events.enabled setting is deprecated. ' \
+                        'Please remove it from your Datadog.configure block and use ' \
+                        'appsec.auto_user_instrumentation.mode instead.'
+                      end
+                    end
+                  end
                 end
 
                 option :mode do |o|
                   o.type :string
                   o.env 'DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING'
-                  o.default 'safe'
+                  o.default SAFE_TRACK_USER_EVENTS_MODE
                   o.setter do |v|
                     if APPSEC_VALID_TRACK_USER_EVENTS_MODE.include?(v)
                       v
                     elsif v == 'disabled'
-                      'safe'
+                      SAFE_TRACK_USER_EVENTS_MODE
                     else
                       Datadog.logger.warn(
                         'The appsec.track_user_events.mode value provided is not supported.' \
-                        'Supported values are: safe | extended.' \
-                        'Using default value `safe`'
+                        "Supported values are: #{APPSEC_VALID_TRACK_USER_EVENTS_MODE.join(" | ")}." \
+                        "Using default value: #{SAFE_TRACK_USER_EVENTS_MODE}."
                       )
-                      'safe'
+
+                      SAFE_TRACK_USER_EVENTS_MODE
+                    end
+                  end
+                  o.after_set do |_, _, precedence|
+                    unless precedence == Datadog::Core::Configuration::Option::Precedence::DEFAULT
+                      Core.log_deprecation(key: :appsec_track_user_events_mode) do
+                        'The appsec.track_user_events.mode setting is deprecated. ' \
+                        'Please remove it from your Datadog.configure block and use ' \
+                        'appsec.auto_user_instrumentation.mode instead.'
+                      end
                     end
                   end
                 end
@@ -197,14 +332,6 @@ module Datadog
                 o.type :bool, nilable: true
                 o.env 'DD_APPSEC_SCA_ENABLED'
               end
-
-              settings :standalone do
-                option :enabled do |o|
-                  o.type :bool
-                  o.env 'DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED'
-                  o.default false
-                end
-              end
             end
           end
         end

data/lib/datadog/appsec/context.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require_relative 'metrics'
+
+module Datadog
+  module AppSec
+    # This class accumulates the context over the request life-cycle and exposes
+    # interface sufficient for instrumentation to perform threat detection.
+    class Context
+      ActiveContextError = Class.new(StandardError)
+
+      attr_reader :trace, :span, :events
+
+      class << self
+        def activate(context)
+          raise ArgumentError, 'not a Datadog::AppSec::Context' unless context.instance_of?(Context)
+          raise ActiveContextError, 'another context is active, nested contexts are not supported' if active
+
+          Thread.current[Ext::ACTIVE_CONTEXT_KEY] = context
+        end
+
+        def deactivate
+          active&.finalize
+        ensure
+          Thread.current[Ext::ACTIVE_CONTEXT_KEY] = nil
+        end
+
+        def active
+          Thread.current[Ext::ACTIVE_CONTEXT_KEY]
+        end
+      end
+
+      def initialize(trace, span, security_engine)
+        @trace = trace
+        @span = span
+        @events = []
+        @security_engine = security_engine
+        @waf_runner = security_engine.new_runner
+        @metrics = Metrics::Collector.new
+      end
+
+      def run_waf(persistent_data, ephemeral_data, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
+        result = @waf_runner.run(persistent_data, ephemeral_data, timeout)
+
+        @metrics.record_waf(result)
+        result
+      end
+
+      def run_rasp(type, persistent_data, ephemeral_data, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
+        result = @waf_runner.run(persistent_data, ephemeral_data, timeout)
+
+        Metrics::Telemetry.report_rasp(type, result)
+        @metrics.record_rasp(result)
+
+        result
+      end
+
+      def extract_schema
+        @waf_runner.run({'waf.context.processor' => {'extract-schema' => true}}, {})
+      end
+
+      def export_metrics
+        return if @span.nil?
+
+        Metrics::Exporter.export_waf_metrics(@metrics.waf, @span)
+        Metrics::Exporter.export_rasp_metrics(@metrics.rasp, @span)
+      end
+
+      def finalize
+        @waf_runner.finalize
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/active_record/instrumentation.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require_relative '../../event'
+require_relative '../../security_event'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module ActiveRecord
+        # AppSec module that will be prepended to ActiveRecord adapter
+        module Instrumentation
+          module_function
+
+          def detect_sql_injection(sql, adapter_name)
+            return unless AppSec.rasp_enabled?
+
+            context = AppSec.active_context
+            return unless context
+
+            # libddwaf expects db system to be lowercase,
+            # in case of sqlite adapter, libddwaf expects 'sqlite' as db system
+            db_system = adapter_name.downcase
+            db_system = 'sqlite' if db_system == 'sqlite3'
+
+            ephemeral_data = {
+              'server.db.statement' => sql,
+              'server.db.system' => db_system
+            }
+
+            waf_timeout = Datadog.configuration.appsec.waf_timeout
+            result = context.run_rasp(Ext::RASP_SQLI, {}, ephemeral_data, waf_timeout)
+
+            if result.match?
+              AppSec::Event.tag_and_keep!(context, result)
+
+              context.events.push(
+                AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
+              )
+
+              AppSec::ActionsHandler.handle(result.actions)
+            end
+          end
+
+          # patch for mysql2, sqlite3, and postgres+jdbc adapters in ActiveRecord >= 7.1
+          module InternalExecQueryAdapterPatch
+            def internal_exec_query(sql, *args, **rest)
+              Instrumentation.detect_sql_injection(sql, adapter_name)
+
+              super
+            end
+          end
+
+          # patch for mysql2, sqlite3, and postgres+jdbc adapters in ActiveRecord < 7.1
+          module ExecQueryAdapterPatch
+            def exec_query(sql, *args, **rest)
+              Instrumentation.detect_sql_injection(sql, adapter_name)
+
+              super
+            end
+          end
+
+          # patch for mysql2, sqlite3, and postgres+jdbc db adapters in ActiveRecord 4
+          module Rails4ExecQueryAdapterPatch
+            def exec_query(sql, *args)
+              Instrumentation.detect_sql_injection(sql, adapter_name)
+
+              super
+            end
+          end
+
+          # patch for non-jdbc postgres adapter in ActiveRecord > 4
+          module ExecuteAndClearAdapterPatch
+            def execute_and_clear(sql, *args, **rest)
+              Instrumentation.detect_sql_injection(sql, adapter_name)
+
+              super
+            end
+          end
+
+          # patch for non-jdbc postgres adapter in ActiveRecord 4
+          module Rails4ExecuteAndClearAdapterPatch
+            def execute_and_clear(sql, name, binds)
+              Instrumentation.detect_sql_injection(sql, adapter_name)
+
+              super
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/active_record/integration.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative '../integration'
+require_relative 'patcher'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module ActiveRecord
+        # This class provides helper methods that are used when patching ActiveRecord
+        class Integration
+          include Datadog::AppSec::Contrib::Integration
+
+          MINIMUM_VERSION = Gem::Version.new('4')
+
+          register_as :active_record, auto_patch: true
+
+          def self.version
+            Gem.loaded_specs['activerecord']&.version
+          end
+
+          def self.loaded?
+            !defined?(::ActiveRecord).nil?
+          end
+
+          def self.compatible?
+            super && version >= MINIMUM_VERSION
+          end
+
+          def self.auto_instrument?
+            true
+          end
+
+          def patcher
+            Patcher
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/active_record/patcher.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require_relative 'instrumentation'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module ActiveRecord
+        # AppSec patcher module for ActiveRecord
+        module Patcher
+          module_function
+
+          def patched?
+            Patcher.instance_variable_get(:@patched)
+          end
+
+          def target_version
+            Integration.version
+          end
+
+          def patch
+            # Rails 7.0 intruduced new on-load hooks for sqlite3 and postgresql adapters
+            # The load hook for mysql2 adapter was introduced in Rails 7.1
+            #
+            # If the adapter is not loaded when the :active_record load hook is called,
+            # we need to add a load hook for the adapter
+            ActiveSupport.on_load :active_record do
+              if defined?(::ActiveRecord::ConnectionAdapters::SQLite3Adapter)
+                ::Datadog::AppSec::Contrib::ActiveRecord::Patcher.patch_sqlite3_adapter
+              else
+                ActiveSupport.on_load :active_record_sqlite3adapter do
+                  ::Datadog::AppSec::Contrib::ActiveRecord::Patcher.patch_sqlite3_adapter
+                end
+              end
+
+              if defined?(::ActiveRecord::ConnectionAdapters::Mysql2Adapter)
+                ::Datadog::AppSec::Contrib::ActiveRecord::Patcher.patch_mysql2_adapter
+              else
+                ActiveSupport.on_load :active_record_mysql2adapter do
+                  ::Datadog::AppSec::Contrib::ActiveRecord::Patcher.patch_mysql2_adapter
+                end
+              end
+
+              if defined?(::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter)
+                ::Datadog::AppSec::Contrib::ActiveRecord::Patcher.patch_postgresql_adapter
+              else
+                ActiveSupport.on_load :active_record_postgresqladapter do
+                  ::Datadog::AppSec::Contrib::ActiveRecord::Patcher.patch_postgresql_adapter
+                end
+              end
+            end
+          end
+
+          def patch_sqlite3_adapter
+            instrumentation_module = if ::ActiveRecord.gem_version >= Gem::Version.new('7.1')
+              Instrumentation::InternalExecQueryAdapterPatch
+            elsif ::ActiveRecord.gem_version.segments.first == 4
+              Instrumentation::Rails4ExecQueryAdapterPatch
+            else
+              Instrumentation::ExecQueryAdapterPatch
+            end
+
+            ::ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend(instrumentation_module)
+          end
+
+          def patch_mysql2_adapter
+            instrumentation_module = if ::ActiveRecord.gem_version >= Gem::Version.new('7.1')
+              Instrumentation::InternalExecQueryAdapterPatch
+            elsif ::ActiveRecord.gem_version.segments.first == 4
+              Instrumentation::Rails4ExecQueryAdapterPatch
+            else
+              Instrumentation::ExecQueryAdapterPatch
+            end
+
+            ::ActiveRecord::ConnectionAdapters::Mysql2Adapter.prepend(instrumentation_module)
+          end
+
+          def patch_postgresql_adapter
+            instrumentation_module = if ::ActiveRecord.gem_version.segments.first == 4
+              Instrumentation::Rails4ExecuteAndClearAdapterPatch
+            else
+              Instrumentation::ExecuteAndClearAdapterPatch
+            end
+
+            if defined?(::ActiveRecord::ConnectionAdapters::JdbcAdapter)
+              instrumentation_module = if ::ActiveRecord.gem_version >= Gem::Version.new('7.1')
+                Instrumentation::InternalExecQueryAdapterPatch
+              elsif ::ActiveRecord.gem_version.segments.first == 4
+                Instrumentation::Rails4ExecQueryAdapterPatch
+              else
+                Instrumentation::ExecQueryAdapterPatch
+              end
+            end
+
+            ::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(instrumentation_module)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/auto_instrument.rb

@@ -9,7 +9,7 @@ module Datadog
         def self.patch_all
           integrations = []
 
-          Datadog::AppSec::Contrib::Integration.registry.each do |_name, integration|
+          Datadog::AppSec::Contrib::Integration.registry.each_value do |integration|
             next unless integration.klass.auto_instrument?
 
             integrations << integration.name

data/lib/datadog/appsec/contrib/devise/configuration.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Devise
+        # A temporary configuration module to accomodate new RFC changes.
+        # NOTE: DEV-3 Remove module
+        module Configuration
+          TRACK_USER_EVENTS_CONVERSION_RULES = {
+            AppSec::Configuration::Settings::SAFE_TRACK_USER_EVENTS_MODE =>
+              AppSec::Configuration::Settings::ANONYMIZATION_AUTO_USER_INSTRUMENTATION_MODE,
+            AppSec::Configuration::Settings::EXTENDED_TRACK_USER_EVENTS_MODE =>
+              AppSec::Configuration::Settings::IDENTIFICATION_AUTO_USER_INSTRUMENTATION_MODE
+          }.freeze
+
+          module_function
+
+          # NOTE: DEV-3 Replace method use with `auto_user_instrumentation.enabled?`
+          def auto_user_instrumentation_enabled?
+            appsec = Datadog.configuration.appsec
+            appsec.auto_user_instrumentation.mode
+
+            unless appsec.auto_user_instrumentation.options[:mode].default_precedence?
+              return appsec.auto_user_instrumentation.enabled?
+            end
+
+            appsec.track_user_events.enabled
+          end
+
+          # NOTE: DEV-3 Replace method use with `auto_user_instrumentation.mode`
+          def auto_user_instrumentation_mode
+            appsec = Datadog.configuration.appsec
+
+            # NOTE: Reading both to trigger precedence set
+            appsec.auto_user_instrumentation.mode
+            appsec.track_user_events.mode
+
+            if !appsec.track_user_events.options[:mode].default_precedence? &&
+                appsec.auto_user_instrumentation.options[:mode].default_precedence?
+              return TRACK_USER_EVENTS_CONVERSION_RULES.fetch(
+                appsec.track_user_events.mode, appsec.auto_user_instrumentation.mode
+              )
+            end
+
+            appsec.auto_user_instrumentation.mode
+          end
+        end
+      end
+    end
+  end
+end