ddtrace 0.49.0 → 1.12.1

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require_relative '../../instrumentation/gateway'
+require_relative '../../reactive/operation'
+require_relative '../reactive/set_user'
+
+module Datadog
+  module AppSec
+    module Monitor
+      module Gateway
+        # Watcher for Apssec internal events
+        module Watcher
+          class << self
+            def watch
+              gateway = Instrumentation.gateway
+
+              watch_user_id(gateway)
+            end
+
+            def watch_user_id(gateway = Instrumentation.gateway)
+              gateway.watch('identity.set_user', :appsec) do |stack, user|
+                block = false
+                event = nil
+                scope = Datadog::AppSec.active_scope
+
+                AppSec::Reactive::Operation.new('identity.set_user') do |op|
+                  Monitor::Reactive::SetUser.subscribe(op, scope.processor_context) do |result, _block|
+                    if result.status == :match
+                      # TODO: should this hash be an Event instance instead?
+                      event = {
+                        waf_result: result,
+                        trace: scope.trace,
+                        span: scope.service_entry_span,
+                        user: user,
+                        actions: result.actions
+                      }
+
+                      if scope.service_entry_span
+                        scope.service_entry_span.set_tag('appsec.blocked', 'true') if result.actions.include?('block')
+                        scope.service_entry_span.set_tag('appsec.event', 'true')
+                      end
+
+                      scope.processor_context.events << event
+                    end
+                  end
+
+                  _result, block = Monitor::Reactive::SetUser.publish(op, user)
+                end
+
+                throw(Datadog::AppSec::Ext::INTERRUPT, [nil, [:block, event]]) if block
+
+                ret, res = stack.call(user)
+
+                if event
+                  res ||= []
+                  res << [:monitor, event]
+                end
+
+                [ret, res]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/monitor/reactive/set_user.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Monitor
+      module Reactive
+        # Dispatch data from Datadog::Kit::Identity.set_user to the WAF context
+        module SetUser
+          ADDRESSES = [
+            'usr.id',
+          ].freeze
+          private_constant :ADDRESSES
+
+          def self.publish(op, user)
+            catch(:block) do
+              op.publish('usr.id', user.id)
+
+              nil
+            end
+          end
+
+          def self.subscribe(op, waf_context)
+            op.subscribe(*ADDRESSES) do |*values|
+              Datadog.logger.debug { "reacted to #{ADDRESSES.inspect}: #{values.inspect}" }
+
+              user_id = values[0]
+
+              waf_args = {
+                'usr.id' => user_id,
+              }
+
+              waf_timeout = Datadog::AppSec.settings.waf_timeout
+              result = waf_context.run(waf_args, waf_timeout)
+
+              Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
+
+              case result.status
+              when :match
+                Datadog.logger.debug { "WAF: #{result.inspect}" }
+
+                block = result.actions.include?('block')
+
+                yield [result, block]
+
+                throw(:block, [result, true]) if block
+              when :ok
+                Datadog.logger.debug { "WAF OK: #{result.inspect}" }
+              when :invalid_call
+                Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
+              when :invalid_rule, :invalid_flow, :no_rule
+                Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
+              else
+                Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/monitor.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require_relative 'monitor/gateway/watcher'
+
+module Datadog
+  module AppSec
+    # Monitor for internal AppSec Events
+    module Monitor
+    end
+  end
+end

data/lib/datadog/appsec/processor/rule_loader.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require_relative '../assets'
+
+module Datadog
+  module AppSec
+    class Processor
+      # RuleLoader utility modules
+      # that load appsec rules and data from  settings
+      module RuleLoader
+        class << self
+          def load_rules(ruleset:)
+            begin
+              case ruleset
+              when :recommended, :strict
+                JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset))
+              when :risky
+                Datadog.logger.warn(
+                  'The :risky Application Security Management ruleset has been deprecated and no longer available.'\
+                  'The `:recommended` ruleset will be used instead.'\
+                  'Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block.'
+                )
+                JSON.parse(Datadog::AppSec::Assets.waf_rules(:recommended))
+              when String
+                JSON.parse(File.read(File.expand_path(ruleset)))
+              when File, StringIO
+                JSON.parse(ruleset.read || '').tap { ruleset.rewind }
+              when Hash
+                ruleset
+              else
+                raise ArgumentError, "unsupported value for ruleset setting: #{ruleset.inspect}"
+              end
+            rescue StandardError => e
+              Datadog.logger.error do
+                "libddwaf ruleset failed to load, ruleset: #{ruleset.inspect} error: #{e.inspect}"
+              end
+
+              nil
+            end
+          end
+
+          def load_data(ip_denylist: [], user_id_denylist: [])
+            data = []
+            data << [denylist_data('blocked_ips', ip_denylist)] if ip_denylist.any?
+            data << [denylist_data('blocked_users', user_id_denylist)] if user_id_denylist.any?
+
+            data
+          end
+
+          private
+
+          def denylist_data(id, denylist)
+            {
+              'id' => id,
+              'type' => 'data_with_expiration',
+              'data' => denylist.map { |v| { 'value' => v.to_s, 'expiration' => 2**63 } }
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/processor/rule_merger.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    class Processor
+      # RuleMerger merge different sources of information
+      # into the rules payload
+      module RuleMerger
+        # RuleVersionMismatchError
+        class RuleVersionMismatchError < StandardError
+          def initialize(version1, version2)
+            msg = 'Merging rule files with different version could lead to unkown behaviour. '\
+              "We have receieve two rule files with versions: #{version1}, #{version2}. "\
+              'Please validate the configuration is correct and try again.'
+            super(msg)
+          end
+        end
+
+        class << self
+          def merge(rules:, data: [], overrides: [], exclusions: [], custom_rules: [])
+            combined_rules = combine_rules(rules)
+
+            combined_data = combine_data(data) if data.any?
+            combined_overrides = combine_overrides(overrides) if overrides.any?
+            combined_exclusions = combine_exclusions(exclusions) if exclusions.any?
+            combined_custom_rules = combine_custom_rules(custom_rules) if custom_rules.any?
+
+            combined_rules['rules_data'] = combined_data if combined_data
+            combined_rules['rules_override'] = combined_overrides if combined_overrides
+            combined_rules['exclusions'] = combined_exclusions if combined_exclusions
+            combined_rules['custom_rules'] = combined_custom_rules if combined_custom_rules
+
+            combined_rules
+          end
+
+          private
+
+          def combine_rules(rules)
+            return rules[0].dup if rules.size == 1
+
+            final_rules = []
+            # @type var final_version: ::String
+            final_version = (_ = nil)
+
+            rules.each do |rule_file|
+              version = rule_file['version']
+
+              if version && !final_version
+                final_version = version
+              elsif final_version != version
+                raise RuleVersionMismatchError.new(final_version, version)
+              end
+
+              final_rules.concat(rule_file['rules'])
+            end
+
+            {
+              'version' => final_version,
+              'rules' => final_rules
+            }
+          end
+
+          def combine_data(data)
+            result = []
+
+            data.each do |data_entry|
+              data_entry.each do |value|
+                existing_data = result.find { |x| x['id'] == value['id'] }
+
+                if existing_data && existing_data['type'] == value['type']
+                  # Duplicate entry base on type and id
+                  # We need to merge the existing data with the new one
+                  # and make sure to remove duplicates
+                  merged_data = merge_data_base_on_expiration(existing_data['data'], value['data'])
+                  existing_data['data'] = merged_data
+                else
+                  result << value
+                end
+              end
+            end
+
+            return unless result.any?
+
+            result
+          end
+
+          def merge_data_base_on_expiration(data1, data2)
+            result = data1.each_with_object({}) do |value, acc|
+              acc[value['value']] = value['expiration']
+            end
+
+            data2.each do |data|
+              if result.key?(data['value'])
+                # The value is duplicated so we need to keep
+                # the one with the highest expiration value
+                # We replace it if the expiration is higher than the current one
+                # or if no experiration
+                current_expiration = result[data['value']]
+                new_expiration = data['expiration']
+
+                if new_expiration.nil? || current_expiration && new_expiration > current_expiration
+                  result[data['value']] = new_expiration
+                end
+              else
+                result[data['value']] = data['expiration']
+              end
+            end
+
+            result.each_with_object([]) do |entry, acc|
+              value = { 'value' => entry[0] }
+              value['expiration'] = entry[1] if entry[1]
+
+              acc << value
+            end
+          end
+
+          def combine_overrides(overrides)
+            overrides.flatten
+          end
+
+          def combine_exclusions(exclusions)
+            exclusions.flatten
+          end
+
+          def combine_custom_rules(custom_rules)
+            custom_rules.flatten
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/processor.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # Processor integrates libddwaf into datadog/appsec
+    class Processor
+      # Context manages a sequence of runs
+      class Context
+        attr_reader :time_ns, :time_ext_ns, :timeouts, :events
+
+        def initialize(processor)
+          @context = Datadog::AppSec::WAF::Context.new(processor.send(:handle))
+          @time_ns = 0.0
+          @time_ext_ns = 0.0
+          @timeouts = 0
+          @events = []
+          @run_mutex = Mutex.new
+        end
+
+        def run(input, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
+          @run_mutex.lock
+
+          start_ns = Core::Utils::Time.get_time(:nanosecond)
+
+          # this WAF::Context#run call is not thread safe as it mutates the context
+          # TODO: remove multiple assignment
+          _code, res = @context.run(input, timeout)
+
+          stop_ns = Core::Utils::Time.get_time(:nanosecond)
+
+          # these updates are not thread safe and should be protected
+          @time_ns += res.total_runtime
+          @time_ext_ns += (stop_ns - start_ns)
+          @timeouts += 1 if res.timeout
+
+          res
+        ensure
+          @run_mutex.unlock
+        end
+
+        def finalize
+          @context.finalize
+        end
+      end
+
+      attr_reader :ruleset_info, :addresses
+
+      def initialize(ruleset:)
+        @ruleset_info = nil
+        @addresses = []
+        settings = Datadog::AppSec.settings
+
+        unless load_libddwaf && create_waf_handle(settings, ruleset)
+          Datadog.logger.warn { 'AppSec is disabled, see logged errors above' }
+        end
+      end
+
+      def ready?
+        !@handle.nil?
+      end
+
+      def finalize
+        @handle.finalize
+      end
+
+      protected
+
+      attr_reader :handle
+
+      private
+
+      def load_libddwaf
+        Processor.require_libddwaf && Processor.libddwaf_provides_waf?
+      end
+
+      def create_waf_handle(settings, ruleset)
+        # TODO: this may need to be reset if the main Datadog logging level changes after initialization
+        Datadog::AppSec::WAF.logger = Datadog.logger if Datadog.logger.debug? && settings.waf_debug
+
+        obfuscator_config = {
+          key_regex: settings.obfuscator_key_regex,
+          value_regex: settings.obfuscator_value_regex,
+        }
+
+        @handle = Datadog::AppSec::WAF::Handle.new(ruleset, obfuscator: obfuscator_config)
+        @ruleset_info = @handle.ruleset_info
+        @addresses = @handle.required_addresses
+
+        true
+      rescue WAF::LibDDWAF::Error => e
+        Datadog.logger.error do
+          "libddwaf failed to initialize, error: #{e.inspect}"
+        end
+
+        @ruleset_info = e.ruleset_info if e.ruleset_info
+
+        false
+      rescue StandardError => e
+        Datadog.logger.error do
+          "libddwaf failed to initialize, error: #{e.inspect}"
+        end
+
+        false
+      end
+
+      class << self
+        # check whether libddwaf is required *and* able to provide the needed feature
+        def libddwaf_provides_waf?
+          defined?(Datadog::AppSec::WAF) ? true : false
+        end
+
+        # libddwaf raises a LoadError on unsupported platforms; it may at some
+        # point succeed in being required yet not provide a specific needed feature.
+        def require_libddwaf
+          Datadog.logger.debug { "libddwaf platform: #{libddwaf_platform}" }
+
+          require 'libddwaf'
+
+          true
+        rescue LoadError => e
+          Datadog.logger.error do
+            'libddwaf failed to load,' \
+              "installed platform: #{libddwaf_platform} ruby platforms: #{ruby_platforms} error: #{e.inspect}"
+          end
+
+          false
+        end
+
+        def libddwaf_spec
+          Gem.loaded_specs['libddwaf']
+        end
+
+        def libddwaf_platform
+          libddwaf_spec ? libddwaf_spec.platform.to_s : 'unknown'
+        end
+
+        def ruby_platforms
+          Gem.platforms.map(&:to_s)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/rate_limiter.rb

@@ -0,0 +1,58 @@
+module Datadog
+  module AppSec
+    # Simple per-thread rate limiter
+    # Since AppSec marks sampling to keep on a security event, this limits the flood of egress traces involving AppSec
+    class RateLimiter
+      def initialize(rate)
+        @rate = rate
+        @timestamps = []
+      end
+
+      def limit
+        now = Time.now.to_f
+
+        loop do
+          oldest = @timestamps.first
+
+          break if oldest.nil? || now - oldest < 1
+
+          @timestamps.shift
+        end
+
+        @timestamps << now
+
+        if (count = @timestamps.count) <= @rate
+          yield
+        else
+          Datadog.logger.debug { "Rate limit hit: #{count}/#{@rate} AppSec traces/second" }
+        end
+      end
+
+      class << self
+        def limit(name, &block)
+          rate_limiter(name).limit(&block)
+        end
+
+        # reset a rate limiter: used for testing
+        def reset!(name)
+          Thread.current[:datadog_security_trace_rate_limiter] = nil
+        end
+
+        protected
+
+        def rate_limiter(name)
+          case name
+          when :traces
+            Thread.current[:datadog_security_trace_rate_limiter] ||= RateLimiter.new(trace_rate_limit)
+          else
+            raise "unsupported rate limiter: #{name.inspect}"
+          end
+        end
+
+        def trace_rate_limit
+          Datadog::AppSec.settings.trace_rate_limit
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/reactive/address_hash.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Reactive
+      # AddressHash for Reactive Engine
+      class AddressHash < Hash
+        def self.new(*arguments, &block)
+          super { |h, k| h[k] = [] }
+        end
+
+        def addresses
+          keys.flatten
+        end
+
+        def with(address)
+          keys.select { |k| k.include?(address) }
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/reactive/engine.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require_relative 'address_hash'
+require_relative 'subscriber'
+
+module Datadog
+  module AppSec
+    module Reactive
+      # Reactive Engine
+      class Engine
+        def initialize
+          @data = {}
+          @subscribers = AddressHash.new
+        end
+
+        def subscribe(*addresses, &block)
+          @subscribers[addresses.freeze] << Subscriber.new(&block)
+        end
+
+        def publish(address, value)
+          # check if someone has address subscribed
+          if @subscribers.addresses.include?(address)
+
+            # someone will be interested, set value
+            @data[address] = value
+
+            # find candidates i.e address groups that contain the just posted address
+            @subscribers.with(address).each do |addresses|
+              # find targets to the address group containing the posted address
+              subscribers = @subscribers[addresses]
+
+              # is all data for the targets available?
+              if (addresses - @data.keys).empty?
+                hash = addresses.each_with_object({}) { |a, h| h[a] = @data[a] }
+                subscribers.each { |s| s.call(*hash.values) }
+              end
+            end
+          end
+        end
+
+        private
+
+        attr_reader :subscribers, :data
+      end
+    end
+  end
+end

data/lib/datadog/appsec/reactive/operation.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require_relative 'engine'
+
+module Datadog
+  module AppSec
+    module Reactive
+      # Reactive Engine nested operation tracking
+      class Operation
+        attr_reader :reactive,
+          :parent,
+          :name
+
+        def initialize(name, parent = nil, reactive_engine = nil)
+          Datadog.logger.debug { "operation: #{name} initialize" }
+          @name = name
+          @parent = parent
+          @reactive = select_reactive_engine(reactive_engine, parent)
+
+          # TODO: concurrent store
+          # TODO: constant
+          Thread.current[:datadog_security_active_operation] = self
+
+          yield self if block_given?
+        ensure
+          finalize
+        end
+
+        # TODO: use structs instead of an arg splat
+        def subscribe(*addresses, &block)
+          reactive.subscribe(*addresses, &block)
+        end
+
+        def publish(address, data)
+          reactive.publish(address, data)
+        end
+
+        def finalize
+          Datadog.logger.debug { "operation: #{name} finalize" }
+          Thread.current[:datadog_security_active_operation] = parent
+        end
+
+        private
+
+        def select_reactive_engine(reactive, parent)
+          return reactive if reactive
+
+          return parent.reactive unless parent.nil?
+
+          Reactive::Engine.new
+        end
+
+        class << self
+          def active
+            Thread.current[:datadog_security_active_operation]
+          end
+
+          private
+
+          # For testing only.
+          def reset!
+            Thread.current[:datadog_security_active_operation] = nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/reactive/subscriber.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Reactive
+      # Reactive Engine subscriber
+      class Subscriber
+        def initialize(&block)
+          @block = block
+          freeze
+        end
+
+        def call(*args)
+          @block.call(*args)
+        end
+      end
+    end
+  end
+end