ddtrace 1.4.2 → 1.19.0

data/lib/datadog/appsec/assets.rb

@@ -1,5 +1,3 @@
-# typed: false
-
 require 'pathname'
 
 module Datadog
@@ -12,8 +10,16 @@ module Datadog
         read("waf_rules/#{kind}.json")
       end
 
-      def blocked
-        @blocked ||= read('blocked.html')
+      def waf_processors
+        read('waf_rules/processors.json')
+      end
+
+      def waf_scanners
+        read('waf_rules/scanners.json')
+      end
+
+      def blocked(format: :html)
+        (@blocked ||= {})[format] ||= read("blocked.#{format}")
       end
 
       def path

data/lib/datadog/appsec/autoload.rb

@@ -1,18 +1,11 @@
-# typed: ignore
-
 if %w[1 true].include?((ENV['DD_APPSEC_ENABLED'] || '').downcase)
-  require_relative '../../ddtrace'
-
-  begin
-    require_relative '../appsec'
-  rescue StandardError => e
-    puts "AppSec failed to load. No security check will be performed. error: #{e.class.name} #{e.message}"
-  end
-
   begin
     require_relative 'contrib/auto_instrument'
     Datadog::AppSec::Contrib::AutoInstrument.patch_all
   rescue StandardError => e
-    puts "AppSec failed to instrument. No security check will be performed. error: #{e.class.name} #{e.message}"
+    Kernel.warn(
+      '[ddtrace] AppSec failed to instrument. No security check will be performed. error: ' \
+      " #{e.class.name} #{e.message}"
+    )
   end
 end

data/lib/datadog/appsec/component.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require_relative 'processor'
+require_relative 'processor/rule_merger'
+require_relative 'processor/rule_loader'
+require_relative 'processor/actions'
+
+module Datadog
+  module AppSec
+    # Core-pluggable component for AppSec
+    class Component
+      class << self
+        def build_appsec_component(settings)
+          return unless settings.respond_to?(:appsec) && settings.appsec.enabled
+
+          processor = create_processor(settings)
+
+          # We want to always instrument user events when AppSec is enabled.
+          # There could be cases in which users use the DD_APPSEC_ENABLED Env variable to
+          # enable AppSec, in that case, Devise is already instrumented.
+          # In the case that users do not use DD_APPSEC_ENABLED, we have to instrument it,
+          # hence the lines above.
+          devise_integration = Datadog::AppSec::Contrib::Devise::Integration.new
+          settings.appsec.instrument(:devise) unless devise_integration.patcher.patched?
+
+          new(processor: processor)
+        end
+
+        private
+
+        def create_processor(settings)
+          rules = AppSec::Processor::RuleLoader.load_rules(ruleset: settings.appsec.ruleset)
+          return nil unless rules
+
+          actions = rules['actions']
+
+          AppSec::Processor::Actions.merge(actions) if actions
+
+          data = AppSec::Processor::RuleLoader.load_data(
+            ip_denylist: settings.appsec.ip_denylist,
+            user_id_denylist: settings.appsec.user_id_denylist,
+          )
+
+          exclusions = AppSec::Processor::RuleLoader.load_exclusions(ip_passlist: settings.appsec.ip_passlist)
+
+          ruleset = AppSec::Processor::RuleMerger.merge(
+            rules: [rules],
+            data: data,
+            exclusions: exclusions,
+          )
+
+          processor = Processor.new(ruleset: ruleset)
+          return nil unless processor.ready?
+
+          processor
+        end
+      end
+
+      attr_reader :processor
+
+      def initialize(processor:)
+        @processor = processor
+        @mutex = Mutex.new
+      end
+
+      def reconfigure(ruleset:, actions:)
+        @mutex.synchronize do
+          AppSec::Processor::Actions.merge(actions)
+
+          new = Processor.new(ruleset: ruleset)
+
+          if new && new.ready?
+            old = @processor
+            @processor = new
+            old.finalize if old
+          end
+        end
+      end
+
+      def reconfigure_lock(&block)
+        @mutex.synchronize(&block)
+      end
+
+      def shutdown!
+        @mutex.synchronize do
+          if processor && processor.ready?
+            processor.finalize
+            @processor = nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/configuration/settings.rb

@@ -1,194 +1,202 @@
-# typed: false
+# frozen_string_literal: true
+
+require_relative '../../core/utils/duration'
+require_relative '../sample_rate'
 
 module Datadog
   module AppSec
     module Configuration
-      # Configuration settings, acting as an integration registry
-      # TODO: as with Configuration, this is a trivial implementation
-      # rubocop:disable Metrics/ClassLength
-      class Settings
-        class << self
-          def boolean
-            lambda do |v|
-              case v
-              when /(1|true)/i
-                true
-              when /(0|false)/i, nil
-                false
-              else
-                raise ArgumentError, "invalid boolean: #{v.inspect}"
-              end
-            end
-          end
-
-          # TODO: allow symbols
-          def string
-            ->(v) { v.to_s }
-          end
+      # Settings
+      module Settings
+        # rubocop:disable Layout/LineLength
+        DEFAULT_OBFUSCATOR_KEY_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?)key)|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization'
+        DEFAULT_OBFUSCATOR_VALUE_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\s*=[^;]|"\s*:\s*"[^"]+")|bearer\s+[a-z0-9\._\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\w=-]+\.ey[I-L][\w=-]+(?:\.[\w.+\/=-]+)?|[\-]{5}BEGIN[a-z\s]+PRIVATE\sKEY[\-]{5}[^\-]+[\-]{5}END[a-z\s]+PRIVATE\sKEY|ssh-rsa\s*[a-z0-9\/\.+]{100,}'
+        # rubocop:enable Layout/LineLength
+        APPSEC_VALID_TRACK_USER_EVENTS_MODE = [
+          'safe',
+          'extended'
+        ].freeze
+        APPSEC_VALID_TRACK_USER_EVENTS_ENABLED_VALUES = [
+          '1',
+          'true'
+        ].concat(APPSEC_VALID_TRACK_USER_EVENTS_MODE).freeze
+
+        def self.extended(base)
+          base = base.singleton_class unless base.is_a?(Class)
+          add_settings!(base)
+        end
 
-          def integer
-            lambda do |v|
-              case v
-              when /(\d+)/
-                Integer(Regexp.last_match[1])
-              else
-                raise ArgumentError, "invalid integer: #{v.inspect}"
+        # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+        def self.add_settings!(base)
+          base.class_eval do
+            settings :appsec do
+              option :enabled do |o|
+                o.type :bool
+                o.env 'DD_APPSEC_ENABLED'
+                o.default false
               end
-            end
-          end
 
-          # rubocop:disable Metrics/MethodLength
-          def duration(base = :ns, type = :integer)
-            lambda do |v|
-              cast = case type
-                     when :integer, Integer
-                       method(:Integer)
-                     when :float, Float
-                       method(:Float)
-                     else
-                       raise ArgumentError, "invalid type: #{v.inspect}"
-                     end
-
-              scale = case base
-                      when :s
-                        1_000_000_000
-                      when :ms
-                        1_000_000
-                      when :us
-                        1000
-                      when :ns
-                        1
-                      else
-                        raise ArgumentError, "invalid base: #{v.inspect}"
-                      end
+              define_method(:instrument) do |integration_name|
+                if enabled
+                  registered_integration = Datadog::AppSec::Contrib::Integration.registry[integration_name]
+                  if registered_integration
+                    klass = registered_integration.klass
+                    if klass.loaded? && klass.compatible?
+                      instance = klass.new
+                      instance.patcher.patch unless instance.patcher.patched?
+                    end
+                  end
+                end
+              end
 
-              case v
-              when /^(\d+)h$/
-                cast.call(Regexp.last_match[1]) * 1_000_000_000 * 60 * 60 / scale
-              when /^(\d+)m$/
-                cast.call(Regexp.last_match[1]) * 1_000_000_000 * 60 / scale
-              when /^(\d+)s$/
-                cast.call(Regexp.last_match[1]) * 1_000_000_000 / scale
-              when /^(\d+)ms$/
-                cast.call(Regexp.last_match[1]) * 1_000_000 / scale
-              when /^(\d+)us$/
-                cast.call(Regexp.last_match[1]) * 1_000 / scale
-              when /^(\d+)ns$/
-                cast.call(Regexp.last_match[1]) / scale
-              when /^(\d+)$/
-                cast.call(Regexp.last_match[1])
-              else
-                raise ArgumentError, "invalid duration: #{v.inspect}"
+              option :ruleset do |o|
+                o.env 'DD_APPSEC_RULES'
+                o.default :recommended
               end
-            end
-          end
-          # rubocop:enable Metrics/MethodLength
-        end
 
-        # rubocop:disable Layout/LineLength
-        DEFAULT_OBFUSCATOR_KEY_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?)key)|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization'.freeze
-        DEFAULT_OBFUSCATOR_VALUE_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\s*=[^;]|"\s*:\s*"[^"]+")|bearer\s+[a-z0-9\._\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\w=-]+\.ey[I-L][\w=-]+(?:\.[\w.+\/=-]+)?|[\-]{5}BEGIN[a-z\s]+PRIVATE\sKEY[\-]{5}[^\-]+[\-]{5}END[a-z\s]+PRIVATE\sKEY|ssh-rsa\s*[a-z0-9\/\.+]{100,}'.freeze
-        # rubocop:enable Layout/LineLength
+              option :ip_passlist do |o|
+                o.default []
+              end
 
-        DEFAULTS = {
-          enabled: false,
-          ruleset: :recommended,
-          waf_timeout: 5_000, # us
-          waf_debug: false,
-          trace_rate_limit: 100, # traces/s
-          obfuscator_key_regex: DEFAULT_OBFUSCATOR_KEY_REGEX,
-          obfuscator_value_regex: DEFAULT_OBFUSCATOR_VALUE_REGEX,
-        }.freeze
-
-        ENVS = {
-          'DD_APPSEC_ENABLED' => [:enabled, Settings.boolean],
-          'DD_APPSEC_RULES' => [:ruleset, Settings.string],
-          'DD_APPSEC_WAF_TIMEOUT' => [:waf_timeout, Settings.duration(:us)],
-          'DD_APPSEC_WAF_DEBUG' => [:waf_debug, Settings.boolean],
-          'DD_APPSEC_TRACE_RATE_LIMIT' => [:trace_rate_limit, Settings.integer],
-          'DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP' => [:obfuscator_key_regex, Settings.string],
-          'DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP' => [:obfuscator_value_regex, Settings.string],
-        }.freeze
-
-        Integration = Struct.new(:integration, :options)
-
-        def initialize
-          @integrations = []
-          @options = DEFAULTS.dup.tap do |options|
-            ENVS.each do |env, (key, conv)|
-              options[key] = conv.call(ENV[env]) if ENV[env]
-            end
-          end
-        end
+              option :ip_denylist do |o|
+                o.type :array
+                o.default []
+              end
 
-        def enabled
-          @options[:enabled]
-        end
+              option :user_id_denylist do |o|
+                o.type :array
+                o.default []
+              end
 
-        def ruleset
-          @options[:ruleset]
-        end
+              option :waf_timeout do |o|
+                o.env 'DD_APPSEC_WAF_TIMEOUT' # us
+                o.default 5_000
+                o.setter do |v|
+                  Datadog::Core::Utils::Duration.call(v.to_s, base: :us)
+                end
+              end
 
-        def waf_timeout
-          @options[:waf_timeout]
-        end
+              option :waf_debug do |o|
+                o.env 'DD_APPSEC_WAF_DEBUG'
+                o.default false
+                o.type :bool
+              end
 
-        def waf_debug
-          @options[:waf_debug]
-        end
+              option :trace_rate_limit do |o|
+                o.type :int
+                o.env 'DD_APPSEC_TRACE_RATE_LIMIT' # trace/s
+                o.default 100
+              end
 
-        def trace_rate_limit
-          @options[:trace_rate_limit]
-        end
+              option :obfuscator_key_regex do |o|
+                o.type :string
+                o.env 'DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP'
+                o.default DEFAULT_OBFUSCATOR_KEY_REGEX
+              end
 
-        def obfuscator_key_regex
-          @options[:obfuscator_key_regex]
-        end
+              option :obfuscator_value_regex do |o|
+                o.type :string
+                o.env 'DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP'
+                o.default DEFAULT_OBFUSCATOR_VALUE_REGEX
+              end
 
-        def obfuscator_value_regex
-          @options[:obfuscator_value_regex]
-        end
+              settings :block do
+                settings :templates do
+                  option :html do |o|
+                    o.env 'DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML'
+                    o.type :string, nilable: true
+                    o.setter do |value|
+                      if value
+                        raise(ArgumentError, "appsec.templates.html: file not found: #{value}") unless File.exist?(value)
 
-        def [](integration_name)
-          integration = Datadog::AppSec::Contrib::Integration.registry[integration_name]
+                        File.open(value, 'rb', &:read) || ''
+                      end
+                    end
+                  end
 
-          raise ArgumentError, "'#{integration_name}' is not a valid integration." unless integration
+                  option :json do |o|
+                    o.env 'DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON'
+                    o.type :string, nilable: true
+                    o.setter do |value|
+                      if value
+                        raise(ArgumentError, "appsec.templates.json: file not found: #{value}") unless File.exist?(value)
 
-          integration.options if integration
-        end
+                        File.open(value, 'rb', &:read) || ''
+                      end
+                    end
+                  end
 
-        def merge(dsl)
-          dsl.options.each do |k, v|
-            @options[k] = v unless v.nil?
-          end
+                  option :text do |o|
+                    o.env 'DD_APPSEC_HTTP_BLOCKED_TEMPLATE_TEXT'
+                    o.type :string, nilable: true
+                    o.setter do |value|
+                      if value
+                        raise(ArgumentError, "appsec.templates.text: file not found: #{value}") unless File.exist?(value)
 
-          return self unless @options[:enabled]
+                        File.open(value, 'rb', &:read) || ''
+                      end
+                    end
+                  end
+                end
+              end
 
-          # patcher.patch may call configure again, hence merge might be called again so it needs to be reentrant
-          dsl.instruments.each do |instrument|
-            # TODO: error handling
-            registered_integration = Datadog::AppSec::Contrib::Integration.registry[instrument.name]
-            @integrations << Integration.new(registered_integration, instrument.options)
+              settings :track_user_events do
+                option :enabled do |o|
+                  o.default true
+                  o.type :bool
+                  o.env 'DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING'
+                  o.env_parser do |env_value|
+                    if env_value == 'disabled'
+                      false
+                    else
+                      APPSEC_VALID_TRACK_USER_EVENTS_ENABLED_VALUES.include?(env_value.strip.downcase)
+                    end
+                  end
+                end
+
+                option :mode do |o|
+                  o.type :string
+                  o.env 'DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING'
+                  o.default 'safe'
+                  o.setter do |v|
+                    if APPSEC_VALID_TRACK_USER_EVENTS_MODE.include?(v)
+                      v
+                    elsif v == 'disabled'
+                      'safe'
+                    else
+                      Datadog.logger.warn(
+                        'The appsec.track_user_events.mode value provided is not supported.' \
+                        'Supported values are: safe | extended.' \
+                        'Using default value `safe`'
+                      )
+                      'safe'
+                    end
+                  end
+                end
+              end
 
-            # TODO: move to a separate apply step
-            klass = registered_integration.klass
-            if klass.loaded? && klass.compatible?
-              instance = klass.new
-              instance.patcher.patch
+              settings :api_security do
+                option :enabled do |o|
+                  o.type :bool
+                  o.env 'DD_EXPERIMENTAL_API_SECURITY_ENABLED'
+                  o.default false
+                end
+
+                option :sample_rate do |o|
+                  o.type :float
+                  o.env 'DD_API_SECURITY_REQUEST_SAMPLE_RATE'
+                  o.default 0.1
+                  o.setter do |value|
+                    value = 1 if value > 1
+                    SampleRate.new(value)
+                  end
+                end
+              end
             end
           end
-
-          self
-        end
-
-        private
-
-        # Restore to original state, for testing only.
-        def reset!
-          initialize
         end
+        # rubocop:enable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
       end
-      # rubocop:enable Metrics/ClassLength
     end
   end
 end

data/lib/datadog/appsec/configuration.rb

@@ -1,80 +1,11 @@
-# typed: true
+# frozen_string_literal: true
 
 require_relative 'configuration/settings'
 
 module Datadog
   module AppSec
     # Configuration for AppSec
-    # TODO: this is a trivial implementation, check with shareable code with
-    # tracer and other products
     module Configuration
-      def self.included(base)
-        base.extend(ClassMethods)
-      end
-
-      # Configuration DSL implementation
-      class DSL
-        Instrument = Struct.new(:name, :options)
-        def initialize
-          @instruments = []
-          @options = {}
-        end
-
-        attr_reader :instruments, :options
-
-        def instrument(name, options = {})
-          @instruments << Instrument.new(name, options)
-        end
-
-        def enabled=(value)
-          options[:enabled] = value
-        end
-
-        def ruleset=(value)
-          options[:ruleset] = value
-        end
-
-        # in microseconds
-        def waf_timeout=(value)
-          options[:waf_timeout] = value
-        end
-
-        def waf_debug=(value)
-          options[:waf_debug] = value
-        end
-
-        def trace_rate_limit=(value)
-          options[:trace_rate_limit] = value
-        end
-
-        def obfuscator_key_regex=(value)
-          options[:obfuscator_key_regex] = value
-        end
-
-        def obfuscator_value_regex=(value)
-          options[:obfuscator_value_regex] = value
-        end
-
-        def [](key)
-          found = @instruments.find { |e| e.name == key }
-
-          found.options if found
-        end
-      end
-
-      # class-level methods for Configuration
-      module ClassMethods
-        def configure
-          dsl = DSL.new
-          yield dsl
-          settings.merge(dsl)
-          settings
-        end
-
-        def settings
-          @settings ||= Settings.new
-        end
-      end
     end
   end
 end

data/lib/datadog/appsec/contrib/auto_instrument.rb

@@ -1,4 +1,4 @@
-# typed: true
+# frozen_string_literal: true
 
 module Datadog
   module AppSec
@@ -15,10 +15,8 @@ module Datadog
             integrations << integration.name
           end
 
-          Datadog::AppSec.configure do |c|
-            integrations.each do |integration_name|
-              c.instrument integration_name
-            end
+          integrations.each do |integration_name|
+            Datadog.configuration.appsec.instrument integration_name
           end
         end
       end

data/lib/datadog/appsec/contrib/devise/event.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Devise
+        # Class to extract event information from the resource
+        class Event
+          UUID_REGEX = /^\h{8}-\h{4}-\h{4}-\h{4}-\h{12}$/.freeze
+
+          SAFE_MODE = 'safe'
+          EXTENDED_MODE = 'extended'
+
+          attr_reader :user_id
+
+          def initialize(resource, mode)
+            @resource = resource
+            @mode = mode
+            @user_id = nil
+            @email = nil
+            @username = nil
+
+            extract if @resource
+          end
+
+          def to_h
+            return @event if defined?(@event)
+
+            @event = {}
+            @event[:email] = @email if @email
+            @event[:username] = @username if @username
+            @event
+          end
+
+          private
+
+          def extract
+            @user_id = @resource.id
+
+            case @mode
+            when EXTENDED_MODE
+              @email = @resource.email
+              @username = @resource.username
+            when SAFE_MODE
+              @user_id = nil unless @user_id && @user_id.to_s =~ UUID_REGEX
+            else
+              Datadog.logger.warn(
+                "Invalid automated user evenst mode: `#{@mode}`. "\
+                              'Supported modes are: `safe` and `extended`.'
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/devise/ext.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Devise
+        # Devise integration constants
+        module Ext
+        end
+      end
+    end
+  end
+end