ddtrace 1.0.0 → 1.10.1

data/lib/datadog/appsec/instrumentation/gateway.rb

@@ -1,4 +1,4 @@
-# typed: true
+# frozen_string_literal: true
 
 module Datadog
   module AppSec
@@ -6,6 +6,22 @@ module Datadog
     module Instrumentation
       # Instrumentation gateway implementation
       class Gateway
+        # Instrumentation gateway middleware
+        class Middleware
+          attr_reader :key, :block
+
+          def initialize(key, &block)
+            @key = key
+            @block = block
+          end
+
+          def call(stack, env)
+            @block.call(stack, env)
+          end
+        end
+
+        private_constant :Middleware
+
         def initialize
           @middlewares = Hash.new { |h, k| h[k] = [] }
         end
@@ -13,16 +29,16 @@ module Datadog
         def push(name, env, &block)
           block ||= -> {}
 
-          middlewares = @middlewares[name]
+          middlewares_for_name = middlewares[name]
 
-          return [block.call, nil] if middlewares.empty?
+          return [block.call, nil] if middlewares_for_name.empty?
 
           wrapped = lambda do |_env|
             [block.call, nil]
           end
 
           # TODO: handle exceptions, except for wrapped
-          stack = middlewares.reverse.reduce(wrapped) do |next_, middleware|
+          stack = middlewares_for_name.reverse.reduce(wrapped) do |next_, middleware|
             lambda do |env_|
               middleware.call(next_, env_)
             end
@@ -31,9 +47,13 @@ module Datadog
           stack.call(env)
         end
 
-        def watch(name, &block)
-          @middlewares[name] << block
+        def watch(name, key, &block)
+          @middlewares[name] << Middleware.new(key, &block) unless middlewares[name].any? { |m| m.key == key }
         end
+
+        private
+
+        attr_reader :middlewares
       end
 
       def self.gateway

data/lib/datadog/appsec/instrumentation.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # Instrumentation for AppSec
+    module Instrumentation
+    end
+  end
+end

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require_relative '../../ext'
+require_relative '../../instrumentation/gateway'
+require_relative '../../reactive/operation'
+require_relative '../reactive/set_user'
+
+module Datadog
+  module AppSec
+    module Monitor
+      module Gateway
+        # Watcher for Apssec internal events
+        module Watcher
+          class << self
+            def watch
+              gateway = Instrumentation.gateway
+
+              watch_user_id(gateway)
+            end
+
+            def watch_user_id(gateway = Instrumentation.gateway)
+              gateway.watch('identity.set_user', :appsec) do |stack, user|
+                block = false
+                event = nil
+                waf_context = Datadog::AppSec::Processor.active_context
+
+                AppSec::Reactive::Operation.new('identity.set_user') do |op|
+                  trace = active_trace
+                  span = active_span
+
+                  Monitor::Reactive::SetUser.subscribe(op, waf_context) do |result, _block|
+                    if result.status == :match
+                      # TODO: should this hash be an Event instance instead?
+                      event = {
+                        waf_result: result,
+                        trace: trace,
+                        span: span,
+                        user: user,
+                        actions: result.actions
+                      }
+
+                      span.set_tag('appsec.event', 'true') if span
+
+                      waf_context.events << event
+                    end
+                  end
+
+                  _result, block = Monitor::Reactive::SetUser.publish(op, user)
+                end
+
+                throw(Datadog::AppSec::Ext::INTERRUPT, [nil, [:block, event]]) if block
+
+                ret, res = stack.call(user)
+
+                if event
+                  res ||= []
+                  res << [:monitor, event]
+                end
+
+                [ret, res]
+              end
+            end
+
+            private
+
+            def active_trace
+              # TODO: factor out tracing availability detection
+
+              return unless defined?(Datadog::Tracing)
+
+              Datadog::Tracing.active_trace
+            end
+
+            def active_span
+              # TODO: factor out tracing availability detection
+
+              return unless defined?(Datadog::Tracing)
+
+              Datadog::Tracing.active_span
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/monitor/reactive/set_user.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Monitor
+      module Reactive
+        # Dispatch data from Datadog::Kit::Identity.set_user to the WAF context
+        module SetUser
+          ADDRESSES = [
+            'usr.id',
+          ].freeze
+          private_constant :ADDRESSES
+
+          def self.publish(op, user)
+            catch(:block) do
+              op.publish('usr.id', user.id)
+
+              nil
+            end
+          end
+
+          def self.subscribe(op, waf_context)
+            op.subscribe(*ADDRESSES) do |*values|
+              Datadog.logger.debug { "reacted to #{ADDRESSES.inspect}: #{values.inspect}" }
+
+              user_id = values[0]
+
+              waf_args = {
+                'usr.id' => user_id,
+              }
+
+              waf_timeout = Datadog::AppSec.settings.waf_timeout
+              result = waf_context.run(waf_args, waf_timeout)
+
+              Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
+
+              case result.status
+              when :match
+                Datadog.logger.debug { "WAF: #{result.inspect}" }
+
+                block = result.actions.include?('block')
+
+                yield [result, block]
+
+                throw(:block, [result, true]) if block
+              when :ok
+                Datadog.logger.debug { "WAF OK: #{result.inspect}" }
+              when :invalid_call
+                Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
+              when :invalid_rule, :invalid_flow, :no_rule
+                Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
+              else
+                Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/monitor.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require_relative 'monitor/gateway/watcher'
+
+module Datadog
+  module AppSec
+    # Monitor for internal AppSec Events
+    module Monitor
+    end
+  end
+end

data/lib/datadog/appsec/processor.rb

@@ -1,28 +1,80 @@
-# typed: ignore
-
-require 'datadog/appsec/assets'
+require_relative 'assets'
 
 module Datadog
   module AppSec
     # Processor integrates libddwaf into datadog/appsec
     class Processor
-      # Interface object to check using case .. when
-      module IOLike
-        def read; end
-        def rewind; end
+      # Context manages a sequence of runs
+      class Context
+        attr_reader :time_ns, :time_ext_ns, :timeouts, :events
+
+        def initialize(processor)
+          @context = Datadog::AppSec::WAF::Context.new(processor.send(:handle))
+          @time_ns = 0.0
+          @time_ext_ns = 0.0
+          @timeouts = 0
+          @events = []
+        end
+
+        def run(input, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
+          start_ns = Core::Utils::Time.get_time(:nanosecond)
+
+          # TODO: remove multiple assignment
+          _code, res = _ = @context.run(input, timeout)
+          # @type var res: WAF::Result
+
+          stop_ns = Core::Utils::Time.get_time(:nanosecond)
+
+          @time_ns += res.total_runtime
+          @time_ext_ns += (stop_ns - start_ns)
+          @timeouts += 1 if res.timeout
+
+          res
+        end
+
+        def finalize
+          @context.finalize
+        end
+      end
+
+      class << self
+        def active_context
+          Thread.current[:datadog_current_waf_context]
+        end
+
+        private
+
+        def active_context=(context)
+          unless context.instance_of?(Context)
+            raise ArgumentError,
+              "The context provide: #{context.inspect} is not a Datadog::AppSec::Processor::Context"
+          end
+
+          Thread.current[:datadog_current_waf_context] = context
+        end
 
-        def self.===(other)
-          instance_methods.all? { |meth| other.respond_to?(meth) }
+        def reset_active_context
+          Thread.current[:datadog_current_waf_context] = nil
         end
       end
 
+      class NoActiveContextError < StandardError; end
+      class AlreadyActiveContextError < StandardError; end
+
+      attr_reader :ruleset_info, :addresses
+
       def initialize
-        @ruleset = nil
-        @handle = nil
+        @ruleset_info = nil
+        @addresses = []
+        settings = Datadog::AppSec.settings
 
-        unless load_libddwaf && load_ruleset && create_waf_handle
+        unless load_libddwaf && load_ruleset(settings) && create_waf_handle(settings)
           Datadog.logger.warn { 'AppSec is disabled, see logged errors above' }
+
+          return
         end
+
+        apply_denylist_data(settings)
       end
 
       def ready?
@@ -30,26 +82,82 @@ module Datadog
       end
 
       def new_context
-        Datadog::AppSec::WAF::Context.new(@handle)
+        Context.new(self)
+      end
+
+      def activate_context
+        existing_active_context = Processor.active_context
+        raise AlreadyActiveContextError if existing_active_context
+
+        context = new_context
+        Processor.send(:active_context=, context)
+        context
+      end
+
+      def deactivate_context
+        context = Processor.active_context
+        raise NoActiveContextError unless context
+
+        Processor.send(:reset_active_context)
+        context.finalize
+      end
+
+      def update_rule_data(data)
+        @handle.update_rule_data(data)
+      end
+
+      def toggle_rules(map)
+        @handle.toggle_rules(map)
       end
 
+      def finalize
+        @handle.finalize
+      end
+
+      protected
+
+      attr_reader :handle
+
       private
 
+      def apply_denylist_data(settings)
+        ruledata_setting = []
+        ruledata_setting << denylist_data('blocked_ips', settings.ip_denylist)
+        ruledata_setting << denylist_data('blocked_users', settings.user_id_denylist)
+
+        update_rule_data(ruledata_setting)
+      end
+
+      def denylist_data(id, denylist)
+        {
+          'id' => id,
+          'type' => 'data_with_expiration',
+          'data' => denylist.map { |v| { 'value' => v.to_s, 'expiration' => 2**63 } }
+        }
+      end
+
       def load_libddwaf
         Processor.require_libddwaf && Processor.libddwaf_provides_waf?
       end
 
-      def load_ruleset
-        ruleset_setting = Datadog::AppSec.settings.ruleset
+      def load_ruleset(settings)
+        ruleset_setting = settings.ruleset
 
         begin
           @ruleset = case ruleset_setting
-                     when :recommended, :risky, :strict
+                     when :recommended, :strict
                        JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset_setting))
+                     when :risky
+                       JSON.parse(Datadog::AppSec::Assets.waf_rules(:recommended))
+                       Datadog.logger.warn(
+                         'The :risky Application Security Management ruleset has been deprecated and no longer available.'\
+                         'The `:recommended` ruleset will be used instead.'\
+                         'Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block.'
+                       )
                      when String
                        JSON.parse(File.read(ruleset_setting))
-                     when IOLike
-                       JSON.parse(ruleset_setting.read).tap { ruleset_setting.rewind }
+                     when File, StringIO
+                       JSON.parse(ruleset_setting.read || '').tap { ruleset_setting.rewind }
                      when Hash
                        ruleset_setting
                      else
@@ -66,13 +174,27 @@ module Datadog
         end
       end
 
-      def create_waf_handle
+      def create_waf_handle(settings)
         # TODO: this may need to be reset if the main Datadog logging level changes after initialization
-        Datadog::AppSec::WAF.logger = Datadog.logger if Datadog.logger.debug? && Datadog::AppSec.settings.waf_debug
+        Datadog::AppSec::WAF.logger = Datadog.logger if Datadog.logger.debug? && settings.waf_debug
 
-        @handle = Datadog::AppSec::WAF::Handle.new(@ruleset)
+        obfuscator_config = {
+          key_regex: settings.obfuscator_key_regex,
+          value_regex: settings.obfuscator_value_regex,
+        }
+        @handle = Datadog::AppSec::WAF::Handle.new(@ruleset, obfuscator: obfuscator_config)
+        @ruleset_info = @handle.ruleset_info
+        @addresses = @handle.required_addresses
 
         true
+      rescue WAF::LibDDWAF::Error => e
+        Datadog.logger.error do
+          "libddwaf failed to initialize, error: #{e.inspect}"
+        end
+
+        @ruleset_info = e.ruleset_info if e.ruleset_info
+
+        false
       rescue StandardError => e
         Datadog.logger.error do
           "libddwaf failed to initialize, error: #{e.inspect}"

data/lib/datadog/appsec/rate_limiter.rb

@@ -1,5 +1,3 @@
-# typed: true
-
 module Datadog
   module AppSec
     # Simple per-thread rate limiter
@@ -35,6 +33,11 @@ module Datadog
           rate_limiter(name).limit(&block)
         end
 
+        # reset a rate limiter: used for testing
+        def reset!(name)
+          Thread.current[:datadog_security_trace_rate_limiter] = nil
+        end
+
         protected
 
         def rate_limiter(name)

data/lib/datadog/appsec/reactive/address_hash.rb

@@ -1,10 +1,14 @@
-# typed: true
+# frozen_string_literal: true
 
 module Datadog
   module AppSec
     module Reactive
-      # Address for Reactive Engine
+      # AddressHash for Reactive Engine
       class AddressHash < Hash
+        def self.new(*arguments, &block)
+          super { |h, k| h[k] = [] }
+        end
+
         def addresses
           keys.flatten
         end

data/lib/datadog/appsec/reactive/engine.rb

@@ -1,7 +1,7 @@
-# typed: true
+# frozen_string_literal: true
 
-require 'datadog/appsec/reactive/address_hash'
-require 'datadog/appsec/reactive/subscriber'
+require_relative 'address_hash'
+require_relative 'subscriber'
 
 module Datadog
   module AppSec
@@ -10,20 +10,19 @@ module Datadog
       class Engine
         def initialize
           @data = {}
-          @subscribers = AddressHash.new { |h, k| h[k] = [] } # TODO: move to AddressHash initializer
-          @children = []
+          @subscribers = AddressHash.new
         end
 
         def subscribe(*addresses, &block)
-          @subscribers[addresses.freeze] << Subscriber.new(&block).freeze # TODO: move freeze to Subscriber
+          @subscribers[addresses.freeze] << Subscriber.new(&block)
         end
 
-        def publish(address, data)
+        def publish(address, value)
           # check if someone has address subscribed
           if @subscribers.addresses.include?(address)
 
-            # someone will be interested, set data
-            @data[address] = data
+            # someone will be interested, set value
+            @data[address] = value
 
             # find candidates i.e address groups that contain the just posted address
             @subscribers.with(address).each do |addresses|
@@ -38,6 +37,10 @@ module Datadog
             end
           end
         end
+
+        private
+
+        attr_reader :subscribers, :data
       end
     end
   end

data/lib/datadog/appsec/reactive/operation.rb

@@ -1,7 +1,6 @@
-# typed: true
+# frozen_string_literal: true
 
-require 'datadog/appsec'
-require 'datadog/appsec/reactive/engine'
+require_relative 'engine'
 
 module Datadog
   module AppSec
@@ -9,14 +8,14 @@ module Datadog
       # Reactive Engine nested operation tracking
       class Operation
         attr_reader :reactive,
-                    :parent,
-                    :name
+          :parent,
+          :name
 
         def initialize(name, parent = nil, reactive_engine = nil)
           Datadog.logger.debug { "operation: #{name} initialize" }
           @name = name
           @parent = parent
-          @reactive = reactive_engine || parent && parent.reactive || Reactive::Engine.new
+          @reactive = select_reactive_engine(reactive_engine, parent)
 
           # TODO: concurrent store
           # TODO: constant
@@ -41,10 +40,27 @@ module Datadog
           Thread.current[:datadog_security_active_operation] = parent
         end
 
+        private
+
+        def select_reactive_engine(reactive, parent)
+          return reactive if reactive
+
+          return parent.reactive unless parent.nil?
+
+          Reactive::Engine.new
+        end
+
         class << self
           def active
             Thread.current[:datadog_security_active_operation]
           end
+
+          private
+
+          # For testing only.
+          def reset!
+            Thread.current[:datadog_security_active_operation] = nil
+          end
         end
       end
     end

data/lib/datadog/appsec/reactive/subscriber.rb

@@ -1,4 +1,4 @@
-# typed: true
+# frozen_string_literal: true
 
 module Datadog
   module AppSec
@@ -7,6 +7,7 @@ module Datadog
       class Subscriber
         def initialize(&block)
           @block = block
+          freeze
         end
 
         def call(*args)

data/lib/datadog/appsec/response.rb

@@ -0,0 +1,61 @@
+require_relative 'assets'
+require_relative 'utils/http/media_range'
+
+module Datadog
+  module AppSec
+    # AppSec response
+    class Response
+      attr_reader :status, :headers, :body
+
+      def initialize(status:, headers: {}, body: [])
+        @status = status
+        @headers = headers
+        @body = body
+      end
+
+      def to_rack
+        [status, headers, body]
+      end
+
+      def to_sinatra_response
+        ::Sinatra::Response.new(body, status, headers)
+      end
+
+      def to_action_dispatch_response
+        ::ActionDispatch::Response.new(status, headers, body)
+      end
+
+      class << self
+        def negotiate(env)
+          Response.new(
+            status: 403,
+            headers: { 'Content-Type' => 'text/html' },
+            body: [Datadog::AppSec::Assets.blocked(format: format(env))]
+          )
+        end
+
+        private
+
+        FORMAT_MAP = {
+          'text/html' => :html,
+          'application/json' => :json,
+          'text/plain' => :text,
+        }.freeze
+
+        DEFAULT_FORMAT = :text
+
+        def format(env)
+          return DEFAULT_FORMAT unless env.key?('HTTP_ACCEPT')
+
+          accepted = env['HTTP_ACCEPT'].split(',').map { |m| Utils::HTTP::MediaRange.new(m) }.sort
+
+          accepted.each_with_object(DEFAULT_FORMAT) do |_default, range|
+            format = FORMAT_MAP.keys.find { |type, _format| range === type }
+
+            return FORMAT_MAP[format] if format
+          end
+        end
+      end
+    end
+  end
+end