ddtrace 1.4.2 → 1.19.0

data/lib/datadog/appsec/processor/rule_loader.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+require_relative '../assets'
+
+module Datadog
+  module AppSec
+    class Processor
+      # RuleLoader utility modules
+      # that load appsec rules and data from  settings
+      module RuleLoader
+        class << self
+          def load_rules(ruleset:)
+            begin
+              case ruleset
+              when :recommended, :strict
+                JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset))
+              when :risky
+                Datadog.logger.warn(
+                  'The :risky Application Security Management ruleset has been deprecated and no longer available.'\
+                  'The `:recommended` ruleset will be used instead.'\
+                  'Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block.'
+                )
+                JSON.parse(Datadog::AppSec::Assets.waf_rules(:recommended))
+              when String
+                JSON.parse(File.read(File.expand_path(ruleset)))
+              when File, StringIO
+                JSON.parse(ruleset.read || '').tap { ruleset.rewind }
+              when Hash
+                ruleset
+              else
+                raise ArgumentError, "unsupported value for ruleset setting: #{ruleset.inspect}"
+              end
+            rescue StandardError => e
+              Datadog.logger.error do
+                "libddwaf ruleset failed to load, ruleset: #{ruleset.inspect} error: #{e.inspect}"
+              end
+
+              nil
+            end
+          end
+
+          def load_data(ip_denylist: [], user_id_denylist: [])
+            data = []
+            data << [denylist_data('blocked_ips', ip_denylist)] if ip_denylist.any?
+            data << [denylist_data('blocked_users', user_id_denylist)] if user_id_denylist.any?
+
+            data
+          end
+
+          def load_exclusions(ip_passlist: [])
+            exclusions = []
+            exclusions << [passlist_exclusions(ip_passlist)] if ip_passlist.any?
+
+            exclusions
+          end
+
+          private
+
+          def denylist_data(id, denylist)
+            {
+              'id' => id,
+              'type' => 'data_with_expiration',
+              'data' => denylist.map { |v| { 'value' => v.to_s, 'expiration' => 2**63 } }
+            }
+          end
+
+          def passlist_exclusions(ip_passlist) # rubocop:disable Metrics/MethodLength
+            case ip_passlist
+            when Array
+              pass = ip_passlist
+              monitor = []
+            when Hash
+              pass = ip_passlist[:pass]
+              monitor = ip_passlist[:monitor]
+            else
+              pass = []
+              monitor = []
+            end
+
+            exclusions = []
+
+            exclusions << {
+              'conditions' => [
+                {
+                  'operator' => 'ip_match',
+                  'parameters' => {
+                    'inputs' => [
+                      {
+                        'address' => 'http.client_ip'
+                      }
+                    ],
+                    'list' => pass
+                  }
+                }
+              ],
+              'id' => SecureRandom.uuid,
+            }
+
+            exclusions << {
+              'conditions' => [
+                {
+                  'operator' => 'ip_match',
+                  'parameters' => {
+                    'inputs' => [
+                      {
+                        'address' => 'http.client_ip'
+                      }
+                    ],
+                    'list' => monitor
+                  }
+                }
+              ],
+              'id' => SecureRandom.uuid,
+              'on_match' => 'monitor'
+            }
+
+            exclusions
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/processor/rule_merger.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+require_relative '../assets'
+
+module Datadog
+  module AppSec
+    class Processor
+      # RuleMerger merge different sources of information
+      # into the rules payload
+      module RuleMerger
+        # RuleVersionMismatchError
+        class RuleVersionMismatchError < StandardError
+          def initialize(version1, version2)
+            msg = 'Merging rule files with different version could lead to unkown behaviour. '\
+              "We have receieve two rule files with versions: #{version1}, #{version2}. "\
+              'Please validate the configuration is correct and try again.'
+            super(msg)
+          end
+        end
+
+        DEFAULT_WAF_PROCESSORS = begin
+          JSON.parse(Datadog::AppSec::Assets.waf_processors)
+        rescue StandardError => e
+          Datadog.logger.error { "libddwaf rulemerger failed to parse default waf processors. Error: #{e.inspect}" }
+          []
+        end
+
+        DEFAULT_WAF_SCANNERS = begin
+          JSON.parse(Datadog::AppSec::Assets.waf_scanners)
+        rescue StandardError => e
+          Datadog.logger.error { "libddwaf rulemerger failed to parse default waf scanners. Error: #{e.inspect}" }
+          []
+        end
+
+        class << self
+          def merge(
+            rules:, data: [], overrides: [], exclusions: [], custom_rules: [],
+            processors: DEFAULT_WAF_PROCESSORS, scanners: DEFAULT_WAF_SCANNERS
+          )
+            combined_rules = combine_rules(rules)
+
+            combined_data = combine_data(data) if data.any?
+            combined_overrides = combine_overrides(overrides) if overrides.any?
+            combined_exclusions = combine_exclusions(exclusions) if exclusions.any?
+            combined_custom_rules = combine_custom_rules(custom_rules) if custom_rules.any?
+
+            combined_rules['rules_data'] = combined_data if combined_data
+            combined_rules['rules_override'] = combined_overrides if combined_overrides
+            combined_rules['exclusions'] = combined_exclusions if combined_exclusions
+            combined_rules['custom_rules'] = combined_custom_rules if combined_custom_rules
+            combined_rules['processors'] = processors
+            combined_rules['scanners'] = scanners
+            combined_rules
+          end
+
+          private
+
+          def combine_rules(rules)
+            return rules[0].dup if rules.size == 1
+
+            final_rules = []
+            # @type var final_version: ::String
+            final_version = (_ = nil)
+
+            rules.each do |rule_file|
+              version = rule_file['version']
+
+              if version && !final_version
+                final_version = version
+              elsif final_version != version
+                raise RuleVersionMismatchError.new(final_version, version)
+              end
+
+              final_rules.concat(rule_file['rules'])
+            end
+
+            {
+              'version' => final_version,
+              'rules' => final_rules
+            }
+          end
+
+          def combine_data(data)
+            result = []
+
+            data.each do |data_entry|
+              data_entry.each do |value|
+                existing_data = result.find { |x| x['id'] == value['id'] }
+
+                if existing_data && existing_data['type'] == value['type']
+                  # Duplicate entry base on type and id
+                  # We need to merge the existing data with the new one
+                  # and make sure to remove duplicates
+                  merged_data = merge_data_base_on_expiration(existing_data['data'], value['data'])
+                  existing_data['data'] = merged_data
+                else
+                  result << value
+                end
+              end
+            end
+
+            return unless result.any?
+
+            result
+          end
+
+          def merge_data_base_on_expiration(data1, data2)
+            result = data1.each_with_object({}) do |value, acc|
+              acc[value['value']] = value['expiration']
+            end
+
+            data2.each do |data|
+              if result.key?(data['value'])
+                # The value is duplicated so we need to keep
+                # the one with the highest expiration value
+                # We replace it if the expiration is higher than the current one
+                # or if no experiration
+                current_expiration = result[data['value']]
+                new_expiration = data['expiration']
+
+                if new_expiration.nil? || current_expiration && new_expiration > current_expiration
+                  result[data['value']] = new_expiration
+                end
+              else
+                result[data['value']] = data['expiration']
+              end
+            end
+
+            result.each_with_object([]) do |entry, acc|
+              value = { 'value' => entry[0] }
+              value['expiration'] = entry[1] if entry[1]
+
+              acc << value
+            end
+          end
+
+          def combine_overrides(overrides)
+            overrides.flatten
+          end
+
+          def combine_exclusions(exclusions)
+            exclusions.flatten
+          end
+
+          def combine_custom_rules(custom_rules)
+            custom_rules.flatten
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/processor.rb

@@ -1,21 +1,9 @@
-# typed: ignore
-
-require_relative 'assets'
+# frozen_string_literal: true
 
 module Datadog
   module AppSec
     # Processor integrates libddwaf into datadog/appsec
     class Processor
-      # Interface object to check using case .. when
-      module IOLike
-        def read; end
-        def rewind; end
-
-        def self.===(other)
-          instance_methods.all? { |meth| other.respond_to?(meth) }
-        end
-      end
-
       # Context manages a sequence of runs
       class Context
         attr_reader :time_ns, :time_ext_ns, :timeouts, :events
@@ -26,42 +14,81 @@ module Datadog
           @time_ext_ns = 0.0
           @timeouts = 0
           @events = []
+          @run_mutex = Mutex.new
         end
 
-        def run(*args)
+        def run(input, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
+          @run_mutex.lock
+
           start_ns = Core::Utils::Time.get_time(:nanosecond)
 
-          ret, res = @context.run(*args)
+          input.reject! do |_, v|
+            case v
+            when TrueClass, FalseClass
+              false
+            else
+              v.nil? ? true : v.empty?
+            end
+          end
+
+          _code, res = @context.run(input, timeout)
 
           stop_ns = Core::Utils::Time.get_time(:nanosecond)
 
+          # these updates are not thread safe and should be protected
           @time_ns += res.total_runtime
           @time_ext_ns += (stop_ns - start_ns)
           @timeouts += 1 if res.timeout
 
-          [ret, res]
+          res
+        ensure
+          @run_mutex.unlock
+        end
+
+        def extract_schema
+          return unless extract_schema?
+
+          input = {
+            'waf.context.processor' => {
+              'extract-schema' => true
+            }
+          }
+
+          _code, res = @context.run(input, WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
+
+          res
+        end
+
+        def finalize
+          @context.finalize
+        end
+
+        private
+
+        def extract_schema?
+          Datadog.configuration.appsec.api_security.enabled &&
+            Datadog.configuration.appsec.api_security.sample_rate.sample?
         end
       end
 
-      attr_reader :ruleset_info, :addresses
+      attr_reader :diagnostics, :addresses
 
-      def initialize
-        @ruleset = nil
-        @handle = nil
-        @ruleset_info = nil
-        @addresses = nil
+      def initialize(ruleset:)
+        @diagnostics = nil
+        @addresses = []
+        settings = Datadog.configuration.appsec
 
-        unless load_libddwaf && load_ruleset && create_waf_handle
+        unless load_libddwaf && create_waf_handle(settings, ruleset)
           Datadog.logger.warn { 'AppSec is disabled, see logged errors above' }
         end
       end
 
       def ready?
-        !@ruleset.nil? && !@handle.nil?
+        !@handle.nil?
       end
 
-      def new_context
-        Context.new(self)
+      def finalize
+        @handle.finalize
       end
 
       protected
@@ -74,52 +101,32 @@ module Datadog
         Processor.require_libddwaf && Processor.libddwaf_provides_waf?
       end
 
-      def load_ruleset
-        ruleset_setting = Datadog::AppSec.settings.ruleset
-
-        begin
-          @ruleset = case ruleset_setting
-                     when :recommended, :risky, :strict
-                       JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset_setting))
-                     when String
-                       JSON.parse(File.read(ruleset_setting))
-                     when IOLike
-                       JSON.parse(ruleset_setting.read).tap { ruleset_setting.rewind }
-                     when Hash
-                       ruleset_setting
-                     else
-                       raise ArgumentError, "unsupported value for ruleset setting: #{ruleset_setting.inspect}"
-                     end
-
-          true
-        rescue StandardError => e
-          Datadog.logger.error do
-            "libddwaf ruleset failed to load, ruleset: #{ruleset_setting.inspect} error: #{e.inspect}"
-          end
-
-          false
-        end
-      end
-
-      def create_waf_handle
+      def create_waf_handle(settings, ruleset)
         # TODO: this may need to be reset if the main Datadog logging level changes after initialization
-        Datadog::AppSec::WAF.logger = Datadog.logger if Datadog.logger.debug? && Datadog::AppSec.settings.waf_debug
+        Datadog::AppSec::WAF.logger = Datadog.logger if Datadog.logger.debug? && settings.waf_debug
 
         obfuscator_config = {
-          key_regex: Datadog::AppSec.settings.obfuscator_key_regex,
-          value_regex: Datadog::AppSec.settings.obfuscator_value_regex,
+          key_regex: settings.obfuscator_key_regex,
+          value_regex: settings.obfuscator_value_regex,
         }
-        @handle = Datadog::AppSec::WAF::Handle.new(@ruleset, obfuscator: obfuscator_config)
-        @ruleset_info = @handle.ruleset_info
+
+        @handle = Datadog::AppSec::WAF::Handle.new(ruleset, obfuscator: obfuscator_config)
+        @diagnostics = @handle.diagnostics
         @addresses = @handle.required_addresses
 
         true
-      rescue StandardError => e
+      rescue WAF::LibDDWAF::Error => e
         Datadog.logger.error do
           "libddwaf failed to initialize, error: #{e.inspect}"
         end
 
-        @ruleset_info = e.ruleset_info if e.respond_to?(:ruleset_info)
+        @diagnostics = e.diagnostics if e.diagnostics
+
+        false
+      rescue StandardError => e
+        Datadog.logger.error do
+          "libddwaf failed to initialize, error: #{e.inspect}"
+        end
 
         false
       end

data/lib/datadog/appsec/rate_limiter.rb

@@ -1,5 +1,3 @@
-# typed: true
-
 module Datadog
   module AppSec
     # Simple per-thread rate limiter
@@ -52,7 +50,7 @@ module Datadog
         end
 
         def trace_rate_limit
-          Datadog::AppSec.settings.trace_rate_limit
+          Datadog.configuration.appsec.trace_rate_limit
         end
       end
     end

data/lib/datadog/appsec/reactive/address_hash.rb

@@ -1,10 +1,14 @@
-# typed: true
+# frozen_string_literal: true
 
 module Datadog
   module AppSec
     module Reactive
-      # Address for Reactive Engine
+      # AddressHash for Reactive Engine
       class AddressHash < Hash
+        def self.new(*arguments, &block)
+          super { |h, k| h[k] = [] }
+        end
+
         def addresses
           keys.flatten
         end

data/lib/datadog/appsec/reactive/engine.rb

@@ -1,4 +1,4 @@
-# typed: true
+# frozen_string_literal: true
 
 require_relative 'address_hash'
 require_relative 'subscriber'
@@ -10,20 +10,19 @@ module Datadog
       class Engine
         def initialize
           @data = {}
-          @subscribers = AddressHash.new { |h, k| h[k] = [] } # TODO: move to AddressHash initializer
-          @children = []
+          @subscribers = AddressHash.new
         end
 
         def subscribe(*addresses, &block)
-          @subscribers[addresses.freeze] << Subscriber.new(&block).freeze # TODO: move freeze to Subscriber
+          @subscribers[addresses.freeze] << Subscriber.new(&block)
         end
 
-        def publish(address, data)
+        def publish(address, value)
           # check if someone has address subscribed
           if @subscribers.addresses.include?(address)
 
-            # someone will be interested, set data
-            @data[address] = data
+            # someone will be interested, set value
+            @data[address] = value
 
             # find candidates i.e address groups that contain the just posted address
             @subscribers.with(address).each do |addresses|
@@ -38,6 +37,10 @@ module Datadog
             end
           end
         end
+
+        private
+
+        attr_reader :subscribers, :data
       end
     end
   end

data/lib/datadog/appsec/reactive/operation.rb

@@ -1,4 +1,4 @@
-# typed: true
+# frozen_string_literal: true
 
 require_relative 'engine'
 
@@ -15,7 +15,7 @@ module Datadog
           Datadog.logger.debug { "operation: #{name} initialize" }
           @name = name
           @parent = parent
-          @reactive = reactive_engine || parent && parent.reactive || Reactive::Engine.new
+          @reactive = select_reactive_engine(reactive_engine, parent)
 
           # TODO: concurrent store
           # TODO: constant
@@ -40,10 +40,27 @@ module Datadog
           Thread.current[:datadog_security_active_operation] = parent
         end
 
+        private
+
+        def select_reactive_engine(reactive, parent)
+          return reactive if reactive
+
+          return parent.reactive unless parent.nil?
+
+          Reactive::Engine.new
+        end
+
         class << self
           def active
             Thread.current[:datadog_security_active_operation]
           end
+
+          private
+
+          # For testing only.
+          def reset!
+            Thread.current[:datadog_security_active_operation] = nil
+          end
         end
       end
     end

data/lib/datadog/appsec/reactive/subscriber.rb

@@ -1,4 +1,4 @@
-# typed: true
+# frozen_string_literal: true
 
 module Datadog
   module AppSec
@@ -7,6 +7,7 @@ module Datadog
       class Subscriber
         def initialize(&block)
           @block = block
+          freeze
         end
 
         def call(*args)