ddtrace 1.10.1 → 1.11.0

data/lib/datadog/appsec/component.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require_relative 'processor'
+require_relative 'processor/rule_merger'
+require_relative 'processor/rule_loader'
 
 module Datadog
   module AppSec
@@ -10,14 +12,27 @@ module Datadog
         def build_appsec_component(settings)
           return unless settings.respond_to?(:appsec) && settings.appsec.enabled
 
-          processor = create_processor
+          processor = create_processor(settings)
           new(processor: processor)
         end
 
         private
 
-        def create_processor
-          processor = Processor.new
+        def create_processor(settings)
+          rules = AppSec::Processor::RuleLoader.load_rules(ruleset: settings.appsec.ruleset)
+          return nil unless rules
+
+          data = AppSec::Processor::RuleLoader.load_data(
+            ip_denylist: settings.appsec.ip_denylist,
+            user_id_denylist: settings.appsec.user_id_denylist
+          )
+
+          ruleset = AppSec::Processor::RuleMerger.merge(
+            rules: [rules],
+            data: data,
+          )
+
+          processor = Processor.new(ruleset: ruleset)
           return nil unless processor.ready?
 
           processor
@@ -28,12 +43,31 @@ module Datadog
 
       def initialize(processor:)
         @processor = processor
+        @mutex = Mutex.new
+      end
+
+      def reconfigure(ruleset:)
+        @mutex.synchronize do
+          new = Processor.new(ruleset: ruleset)
+
+          if new && new.ready?
+            old = @processor
+            @processor = new
+            old.finalize if old
+          end
+        end
+      end
+
+      def reconfigure_lock(&block)
+        @mutex.synchronize(&block)
       end
 
       def shutdown!
-        if processor && processor.ready?
-          processor.finalize
-          @processor = nil
+        @mutex.synchronize do
+          if processor && processor.ready?
+            processor.finalize
+            @processor = nil
+          end
         end
       end
     end

data/lib/datadog/appsec/configuration/settings.rb

@@ -1,3 +1,7 @@
+# frozen_string_literal: true
+
+require 'set'
+
 module Datadog
   module AppSec
     module Configuration
@@ -87,8 +91,8 @@ module Datadog
         end
 
         # rubocop:disable Layout/LineLength
-        DEFAULT_OBFUSCATOR_KEY_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?)key)|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization'.freeze
-        DEFAULT_OBFUSCATOR_VALUE_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\s*=[^;]|"\s*:\s*"[^"]+")|bearer\s+[a-z0-9\._\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\w=-]+\.ey[I-L][\w=-]+(?:\.[\w.+\/=-]+)?|[\-]{5}BEGIN[a-z\s]+PRIVATE\sKEY[\-]{5}[^\-]+[\-]{5}END[a-z\s]+PRIVATE\sKEY|ssh-rsa\s*[a-z0-9\/\.+]{100,}'.freeze
+        DEFAULT_OBFUSCATOR_KEY_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?)key)|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization'
+        DEFAULT_OBFUSCATOR_VALUE_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\s*=[^;]|"\s*:\s*"[^"]+")|bearer\s+[a-z0-9\._\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\w=-]+\.ey[I-L][\w=-]+(?:\.[\w.+\/=-]+)?|[\-]{5}BEGIN[a-z\s]+PRIVATE\sKEY[\-]{5}[^\-]+[\-]{5}END[a-z\s]+PRIVATE\sKEY|ssh-rsa\s*[a-z0-9\/\.+]{100,}'
         # rubocop:enable Layout/LineLength
 
         DEFAULTS = {
@@ -116,9 +120,14 @@ module Datadog
 
         def initialize
           @integrations = []
+          # Stores which options have been configured using Datadog.configure block or ENV variables
+          @configured = Set.new
           @options = DEFAULTS.dup.tap do |options|
             ENVS.each do |env, (key, conv)|
-              options[key] = conv.call(ENV[env]) if ENV[env]
+              if ENV[env]
+                options[key] = conv.call(ENV[env])
+                @configured << key
+              end
             end
           end
         end
@@ -182,7 +191,10 @@ module Datadog
 
         def merge(dsl)
           dsl.options.each do |k, v|
-            @options[k] = v unless v.nil?
+            unless v.nil?
+              @options[k] = v
+              @configured << k
+            end
           end
 
           return self unless @options[:enabled]
@@ -206,6 +218,10 @@ module Datadog
 
         private
 
+        def default?(option)
+          !@configured.include?(option)
+        end
+
         # Restore to original state, for testing only.
         def reset!
           initialize

data/lib/datadog/appsec/configuration.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'configuration/settings'
 
 module Datadog
@@ -82,6 +84,12 @@ module Datadog
         def settings
           @settings ||= Settings.new
         end
+
+        private
+
+        def default_setting?(setting)
+          settings.send(:default?, setting)
+        end
       end
     end
   end

data/lib/datadog/appsec/contrib/auto_instrument.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/contrib/configuration/settings.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../../../core/configuration/base'
 
 module Datadog

data/lib/datadog/appsec/contrib/integration.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/contrib/patcher.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/contrib/rack/configuration/settings.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../../configuration/settings'
 require_relative '../ext'
 

data/lib/datadog/appsec/contrib/rack/gateway/request.rb

@@ -14,7 +14,7 @@ module Datadog
             attr_reader :env
 
             def initialize(env)
-              super
+              super()
               @env = env
             end
 
@@ -33,20 +33,9 @@ module Datadog
               end
             end
 
-            # Rack < 2.0 does not have :each_header
-            # TODO: We need access to Rack here. We must make sure we are able to load AppSec without Rack,
-            # TODO: while still ensure correctness in ths code path.
-            if defined?(::Rack) && ::Rack::Request.instance_methods.include?(:each_header)
-              def headers
-                request.each_header.each_with_object({}) do |(k, v), h|
-                  h[k.gsub(/^HTTP_/, '').downcase.tr('_', '-')] = v if k =~ /^HTTP_/
-                end
-              end
-            else
-              def headers
-                request.env.each_with_object({}) do |(k, v), h|
-                  h[k.gsub(/^HTTP_/, '').downcase.tr('_', '-')] = v if k =~ /^HTTP_/
-                end
+            def headers
+              request.env.each_with_object({}) do |(k, v), h|
+                h[k.gsub(/^HTTP_/, '').downcase.tr('_', '-')] = v if k =~ /^HTTP_/
               end
             end
 

data/lib/datadog/appsec/contrib/rack/gateway/response.rb

@@ -12,7 +12,7 @@ module Datadog
             attr_reader :body, :status, :headers, :active_context
 
             def initialize(body, status, headers, active_context:)
-              super
+              super()
               @body = body
               @status = status
               @headers = headers.each_with_object({}) { |(k, v), h| h[k.downcase] = v }

data/lib/datadog/appsec/contrib/rack/patcher.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../patcher'
 require_relative '../../monitor'
 require_relative 'gateway/watcher'

data/lib/datadog/appsec/contrib/rack/reactive/response.rb

@@ -9,12 +9,14 @@ module Datadog
           module Response
             ADDRESSES = [
               'response.status',
+              'response.headers',
             ].freeze
             private_constant :ADDRESSES
 
             def self.publish(op, gateway_response)
               catch(:block) do
                 op.publish('response.status', gateway_response.status)
+                op.publish('response.headers', gateway_response.headers)
 
                 nil
               end
@@ -25,9 +27,13 @@ module Datadog
                 Datadog.logger.debug { "reacted to #{ADDRESSES.inspect}: #{values.inspect}" }
 
                 response_status = values[0]
+                response_headers = values[1]
+                response_headers_no_cookies = response_headers.dup.tap { |h| h.delete('set-cookie') }
 
                 waf_args = {
                   'server.response.status' => response_status.to_s,
+                  'server.response.headers' => response_headers,
+                  'server.response.headers.no_cookies' => response_headers_no_cookies,
                 }
 
                 waf_timeout = Datadog::AppSec.settings.waf_timeout

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -23,17 +23,29 @@ module Datadog
             @oneshot_tags_sent = false
           end
 
+          # rubocop:disable Metrics/PerceivedComplexity,Metrics/CyclomaticComplexity,Metrics/MethodLength
           def call(env)
             return @app.call(env) unless Datadog::AppSec.enabled?
 
-            processor = Datadog::AppSec.processor
+            Datadog::Core::Remote.active_remote.barrier(:once) unless Datadog::Core::Remote.active_remote.nil?
 
-            return @app.call(env) if processor.nil? || !processor.ready?
+            processor = nil
+            ready = false
+            context = nil
+
+            Datadog::AppSec.reconfigure_lock do
+              processor = Datadog::AppSec.processor
+
+              if !processor.nil? && processor.ready?
+                context = processor.activate_context
+                env['datadog.waf.context'] = context
+                ready = true
+              end
+            end
 
             # TODO: handle exceptions, except for @app.call
 
-            context = processor.activate_context
-            env['datadog.waf.context'] = context
+            return @app.call(env) unless ready
 
             gateway_request = Gateway::Request.new(env)
 
@@ -76,6 +88,7 @@ module Datadog
               processor.deactivate_context
             end
           end
+          # rubocop:enable Metrics/PerceivedComplexity,Metrics/CyclomaticComplexity,Metrics/MethodLength
 
           private
 

data/lib/datadog/appsec/contrib/rails/configuration/settings.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../../configuration/settings'
 require_relative '../ext'
 

data/lib/datadog/appsec/contrib/rails/framework.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/contrib/rails/gateway/request.rb

@@ -12,7 +12,7 @@ module Datadog
             attr_reader :request
 
             def initialize(request)
-              super
+              super()
               @request = request
             end
 

data/lib/datadog/appsec/contrib/rails/request.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/contrib/rails/request_middleware.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/contrib/sinatra/configuration/settings.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../../configuration/settings'
 require_relative '../ext'
 

data/lib/datadog/appsec/contrib/sinatra/framework.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/contrib/sinatra/gateway/route_params.rb

@@ -12,7 +12,7 @@ module Datadog
             attr_reader :params
 
             def initialize(params)
-              super
+              super()
               @params = params
             end
           end

data/lib/datadog/appsec/contrib/sinatra/request_middleware.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/extensions.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'configuration'
 
 module Datadog

data/lib/datadog/appsec/instrumentation/gateway/argument.rb

@@ -5,16 +5,14 @@ module Datadog
     module Instrumentation
       class Gateway
         # Base class for Gateway Arguments
-        class Argument
-          def initialize(*); end
-        end
+        class Argument; end # rubocop:disable Lint/EmptyClass
 
         # Gateway User argument
         class User < Argument
           attr_reader :id
 
           def initialize(id)
-            super
+            super()
             @id = id
           end
         end

data/lib/datadog/appsec/processor/rule_loader.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require_relative '../assets'
+
+module Datadog
+  module AppSec
+    class Processor
+      # RuleLoader utility modules
+      # that load appsec rules and data from  settings
+      module RuleLoader
+        class << self
+          def load_rules(ruleset:)
+            begin
+              case ruleset
+              when :recommended, :strict
+                JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset))
+              when :risky
+                Datadog.logger.warn(
+                  'The :risky Application Security Management ruleset has been deprecated and no longer available.'\
+                  'The `:recommended` ruleset will be used instead.'\
+                  'Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block.'
+                )
+                JSON.parse(Datadog::AppSec::Assets.waf_rules(:recommended))
+              when String
+                JSON.parse(File.read(File.expand_path(ruleset)))
+              when File, StringIO
+                JSON.parse(ruleset.read || '').tap { ruleset.rewind }
+              when Hash
+                ruleset
+              else
+                raise ArgumentError, "unsupported value for ruleset setting: #{ruleset.inspect}"
+              end
+            rescue StandardError => e
+              Datadog.logger.error do
+                "libddwaf ruleset failed to load, ruleset: #{ruleset.inspect} error: #{e.inspect}"
+              end
+
+              nil
+            end
+          end
+
+          def load_data(ip_denylist: [], user_id_denylist: [])
+            data = []
+            data << { 'rules_data' => [denylist_data('blocked_ips', ip_denylist)] } if ip_denylist.any?
+            data << { 'rules_data' => [denylist_data('blocked_users', user_id_denylist)] } if user_id_denylist.any?
+
+            data.any? ? data : nil
+          end
+
+          private
+
+          def denylist_data(id, denylist)
+            {
+              'id' => id,
+              'type' => 'data_with_expiration',
+              'data' => denylist.map { |v| { 'value' => v.to_s, 'expiration' => 2**63 } }
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/processor/rule_merger.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    class Processor
+      # RuleMerger merge different sources of information
+      # into the rules payload
+      module RuleMerger
+        # RuleVersionMismatchError
+        class RuleVersionMismatchError < StandardError
+          def initialize(version1, version2)
+            msg = 'Merging rule files with different version could lead to unkown behaviour. '\
+              "We have receieve two rule files with versions: #{version1}, #{version2}. "\
+              'Please validate the configuration is correct and try again.'
+            super(msg)
+          end
+        end
+
+        class << self
+          def merge(rules:, data: nil, overrides: nil, exclusions: nil)
+            combined_rules = combine_rules(rules)
+
+            rules_data = combine_data(data) if data
+            rules_overrides = combine_overrides(overrides) if overrides
+            rules_exclusions = combine_exclusions(exclusions) if exclusions
+
+            combined_rules['rules_data'] = rules_data if rules_data
+            combined_rules['rules_override'] = rules_overrides if rules_overrides
+            combined_rules['exclusions'] = rules_exclusions if rules_exclusions
+
+            combined_rules
+          end
+
+          private
+
+          def combine_rules(rules)
+            return rules[0].dup if rules.size == 1
+
+            final_rules = []
+            # @type var final_version: ::String
+            final_version = (_ = nil)
+
+            rules.each do |rule_file|
+              version = rule_file['version']
+
+              if version && !final_version
+                final_version = version
+              elsif final_version != version
+                raise RuleVersionMismatchError.new(final_version, version)
+              end
+
+              final_rules.concat(rule_file['rules'])
+            end
+
+            {
+              'version' => final_version,
+              'rules' => final_rules
+            }
+          end
+
+          def combine_data(data)
+            result = []
+
+            data.each do |data_entry|
+              data_entry['rules_data'].each do |value|
+                existing_data = result.find { |x| x['id'] == value['id'] }
+
+                if existing_data && existing_data['type'] == value['type']
+                  # Duplicate entry base on type and id
+                  # We need to merge the existing data with the new one
+                  # and make sure to remove duplicates
+                  merged_data = merge_data_base_on_expiration(existing_data['data'], value['data'])
+                  existing_data['data'] = merged_data
+                else
+                  result << value
+                end
+              end
+            end
+
+            return unless result.any?
+
+            result
+          end
+
+          def merge_data_base_on_expiration(data1, data2)
+            result = data1.each_with_object({}) do |value, acc|
+              acc[value['value']] = value['expiration']
+            end
+
+            data2.each do |data|
+              if result.key?(data['value'])
+                # The value is duplicated so we need to keep
+                # the one with the highest expiration value
+                # We replace it if the expiration is higher than the current one
+                # or if no experiration
+                current_expiration = result[data['value']]
+                new_expiration = data['expiration']
+
+                if new_expiration.nil? || current_expiration && new_expiration > current_expiration
+                  result[data['value']] = new_expiration
+                end
+              else
+                result[data['value']] = data['expiration']
+              end
+            end
+
+            result.each_with_object([]) do |entry, acc|
+              value = { 'value' => entry[0] }
+              value['expiration'] = entry[1] if entry[1]
+
+              acc << value
+            end
+          end
+
+          def combine_overrides(overrides)
+            rules_override = []
+
+            overrides.each do |override|
+              override['rules_override'].each do |rule_override|
+                rules_override << rule_override
+              end
+            end
+
+            return if rules_override.empty?
+
+            rules_override
+          end
+
+          def combine_exclusions(exclusions)
+            rules_exclusions = []
+
+            exclusions.each do |exclusion|
+              exclusion['exclusions'].each do |rule_exclusion|
+                rules_exclusions << rule_exclusion
+              end
+            end
+
+            return if rules_exclusions.empty?
+
+            rules_exclusions
+          end
+        end
+      end
+    end
+  end
+end