ddtrace 1.5.0 → 1.5.2

data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb

@@ -16,8 +16,10 @@ module Datadog
           module Watcher
             # rubocop:disable Metrics/AbcSize
             # rubocop:disable Metrics/MethodLength
+            # rubocop:disable Metrics/CyclomaticComplexity
+            # rubocop:disable Metrics/PerceivedComplexity
             def self.watch
-              Instrumentation.gateway.watch('rack.request') do |stack, request|
+              Instrumentation.gateway.watch('rack.request', :appsec) do |stack, request|
                 block = false
                 event = nil
                 waf_context = request.env['datadog.waf.context']
@@ -26,23 +28,24 @@ module Datadog
                   trace = active_trace
                   span = active_span
 
-                  Rack::Reactive::Request.subscribe(op, waf_context) do |action, result, _block|
-                    record = [:block, :monitor].include?(action)
-                    if record
+                  Rack::Reactive::Request.subscribe(op, waf_context) do |result, _block|
+                    if result.status == :match
                       # TODO: should this hash be an Event instance instead?
                       event = {
                         waf_result: result,
                         trace: trace,
                         span: span,
                         request: request,
-                        action: action
+                        actions: result.actions
                       }
 
+                      span.set_tag('appsec.event', 'true') if span
+
                       waf_context.events << event
                     end
                   end
 
-                  _action, _result, block = Rack::Reactive::Request.publish(op, request)
+                  _result, block = Rack::Reactive::Request.publish(op, request)
                 end
 
                 next [nil, [[:block, event]]] if block
@@ -57,7 +60,7 @@ module Datadog
                 [ret, res]
               end
 
-              Instrumentation.gateway.watch('rack.response') do |stack, response|
+              Instrumentation.gateway.watch('rack.response', :appsec) do |stack, response|
                 block = false
                 event = nil
                 waf_context = response.instance_eval { @waf_context }
@@ -66,23 +69,24 @@ module Datadog
                   trace = active_trace
                   span = active_span
 
-                  Rack::Reactive::Response.subscribe(op, waf_context) do |action, result, _block|
-                    record = [:block, :monitor].include?(action)
-                    if record
+                  Rack::Reactive::Response.subscribe(op, waf_context) do |result, _block|
+                    if result.status == :match
                       # TODO: should this hash be an Event instance instead?
                       event = {
                         waf_result: result,
                         trace: trace,
                         span: span,
                         response: response,
-                        action: action
+                        actions: result.actions
                       }
 
+                      span.set_tag('appsec.event', 'true') if span
+
                       waf_context.events << event
                     end
                   end
 
-                  _action, _result, block = Rack::Reactive::Response.publish(op, response)
+                  _result, block = Rack::Reactive::Response.publish(op, response)
                 end
 
                 next [nil, [[:block, event]]] if block
@@ -97,7 +101,7 @@ module Datadog
                 [ret, res]
               end
 
-              Instrumentation.gateway.watch('rack.request.body') do |stack, request|
+              Instrumentation.gateway.watch('rack.request.body', :appsec) do |stack, request|
                 block = false
                 event = nil
                 waf_context = request.env['datadog.waf.context']
@@ -106,23 +110,24 @@ module Datadog
                   trace = active_trace
                   span = active_span
 
-                  Rack::Reactive::RequestBody.subscribe(op, waf_context) do |action, result, _block|
-                    record = [:block, :monitor].include?(action)
-                    if record
+                  Rack::Reactive::RequestBody.subscribe(op, waf_context) do |result, _block|
+                    if result.status == :match
                       # TODO: should this hash be an Event instance instead?
                       event = {
                         waf_result: result,
                         trace: trace,
                         span: span,
                         request: request,
-                        action: action
+                        actions: result.actions
                       }
 
+                      span.set_tag('appsec.event', 'true') if span
+
                       waf_context.events << event
                     end
                   end
 
-                  _action, _result, block = Rack::Reactive::RequestBody.publish(op, request)
+                  _result, block = Rack::Reactive::RequestBody.publish(op, request)
                 end
 
                 next [nil, [[:block, event]]] if block
@@ -137,6 +142,8 @@ module Datadog
                 [ret, res]
               end
             end
+            # rubocop:enable Metrics/CyclomaticComplexity
+            # rubocop:enable Metrics/PerceivedComplexity
             # rubocop:enable Metrics/MethodLength
             # rubocop:enable Metrics/AbcSize
 

data/lib/datadog/appsec/contrib/rack/reactive/request.rb

@@ -54,27 +54,27 @@ module Datadog
                 }
 
                 waf_timeout = Datadog::AppSec.settings.waf_timeout
-                action, result = waf_context.run(waf_args, waf_timeout)
+                result = waf_context.run(waf_args, waf_timeout)
 
                 Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
 
-                # TODO: encapsulate return array in a type
-                case action
-                when :monitor
+                case result.status
+                when :match
                   Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, false]
-                when :block
-                  Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, true]
-                  throw(:block, [action, result, true])
-                when :good
+
+                  block = result.actions.include?('block')
+
+                  yield [result, block]
+
+                  throw(:block, [result, true]) if block
+                when :ok
                   Datadog.logger.debug { "WAF OK: #{result.inspect}" }
                 when :invalid_call
                   Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
                 when :invalid_rule, :invalid_flow, :no_rule
                   Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
                 else
-                  Datadog.logger.debug { "WAF UNKNOWN: #{action.inspect} #{result.inspect}" }
+                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
                 end
               end
             end

data/lib/datadog/appsec/contrib/rack/reactive/request_body.rb

@@ -32,27 +32,27 @@ module Datadog
                 }
 
                 waf_timeout = Datadog::AppSec.settings.waf_timeout
-                action, result = waf_context.run(waf_args, waf_timeout)
+                result = waf_context.run(waf_args, waf_timeout)
 
                 Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
 
-                # TODO: encapsulate return array in a type
-                case action
-                when :monitor
+                case result.status
+                when :match
                   Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, false]
-                when :block
-                  Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, true]
-                  throw(:block, [action, result, true])
-                when :good
+
+                  block = result.actions.include?('block')
+
+                  yield [result, block]
+
+                  throw(:block, [result, true]) if block
+                when :ok
                   Datadog.logger.debug { "WAF OK: #{result.inspect}" }
                 when :invalid_call
                   Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
                 when :invalid_rule, :invalid_flow, :no_rule
                   Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
                 else
-                  Datadog.logger.debug { "WAF UNKNOWN: #{action.inspect} #{result.inspect}" }
+                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
                 end
               end
             end

data/lib/datadog/appsec/contrib/rack/reactive/response.rb

@@ -32,27 +32,27 @@ module Datadog
                 }
 
                 waf_timeout = Datadog::AppSec.settings.waf_timeout
-                action, result = waf_context.run(waf_args, waf_timeout)
+                result = waf_context.run(waf_args, waf_timeout)
 
                 Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
 
-                # TODO: encapsulate return array in a type
-                case action
-                when :monitor
+                case result.status
+                when :match
                   Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, false]
-                when :block
-                  Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, true]
-                  throw(:block, [action, result, true])
-                when :good
+
+                  block = result.actions.include?('block')
+
+                  yield [result, block]
+
+                  throw(:block, [result, true]) if block
+                when :ok
                   Datadog.logger.debug { "WAF OK: #{result.inspect}" }
                 when :invalid_call
                   Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
                 when :invalid_rule, :invalid_flow, :no_rule
                   Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
                 else
-                  Datadog.logger.debug { "WAF UNKNOWN: #{action.inspect} #{result.inspect}" }
+                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
                 end
               end
             end

data/lib/datadog/appsec/contrib/rack/request.rb

@@ -47,6 +47,9 @@ module Datadog
           end
 
           def self.form_hash(request)
+            # force form data processing
+            request.POST if request.form_data?
+
             # usually Hash<String,String> but can be a more complex
             # Hash<String,String||Array||Hash> when e.g coming from JSON
             request.env['rack.request.form_hash']

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -6,6 +6,9 @@ require_relative '../../instrumentation/gateway'
 require_relative '../../processor'
 require_relative '../../assets'
 
+require_relative '../../../tracing/client_ip'
+require_relative '../../../tracing/contrib/rack/header_collection'
+
 module Datadog
   module AppSec
     module Contrib
@@ -21,7 +24,7 @@ module Datadog
           end
 
           def call(env)
-            return @app.call(env) unless @processor.ready?
+            return @app.call(env) unless Datadog.configuration.appsec.enabled && @processor.ready?
 
             # TODO: handle exceptions, except for @app.call
 
@@ -30,7 +33,7 @@ module Datadog
             env['datadog.waf.context'] = context
             request = ::Rack::Request.new(env)
 
-            add_appsec_tags
+            add_appsec_tags(active_trace, active_span, env)
 
             request_return, request_response = Instrumentation.gateway.push('rack.request', request) do
               @app.call(env)
@@ -56,7 +59,8 @@ module Datadog
 
             request_return
           ensure
-            add_waf_runtime_tags(context) if context
+            add_waf_runtime_tags(active_trace, context) if context
+            context.finalize if context
           end
 
           private
@@ -69,41 +73,60 @@ module Datadog
             Datadog::Tracing.active_trace
           end
 
-          def add_appsec_tags
-            return unless active_trace
+          def active_span
+            # TODO: factor out tracing availability detection
+
+            return unless defined?(Datadog::Tracing)
+
+            Datadog::Tracing.active_span
+          end
+
+          def add_appsec_tags(trace, span, env)
+            return unless trace
+
+            trace.set_tag('_dd.appsec.enabled', 1)
+            trace.set_tag('_dd.runtime_family', 'ruby')
+            trace.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
 
-            active_trace.set_tag('_dd.appsec.enabled', 1)
-            active_trace.set_tag('_dd.runtime_family', 'ruby')
-            active_trace.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
+            if span && span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_CLIENT_IP).nil?
+              request_header_collection = Datadog::Tracing::Contrib::Rack::Header::RequestHeaderCollection.new(env)
+
+              # always collect client ip, as this is part of AppSec provided functionality
+              Datadog::Tracing::ClientIp.set_client_ip_tag!(
+                span,
+                headers: request_header_collection,
+                remote_ip: env['REMOTE_ADDR']
+              )
+            end
 
             if @processor.ruleset_info
-              active_trace.set_tag('_dd.appsec.event_rules.version', @processor.ruleset_info[:version])
+              trace.set_tag('_dd.appsec.event_rules.version', @processor.ruleset_info[:version])
 
               unless @oneshot_tags_sent
                 # Small race condition, but it's inoccuous: worst case the tags
                 # are sent a couple of times more than expected
                 @oneshot_tags_sent = true
 
-                active_trace.set_tag('_dd.appsec.event_rules.loaded', @processor.ruleset_info[:loaded].to_f)
-                active_trace.set_tag('_dd.appsec.event_rules.error_count', @processor.ruleset_info[:failed].to_f)
-                active_trace.set_tag('_dd.appsec.event_rules.errors', JSON.dump(@processor.ruleset_info[:errors]))
-                active_trace.set_tag('_dd.appsec.event_rules.addresses', JSON.dump(@processor.addresses))
+                trace.set_tag('_dd.appsec.event_rules.loaded', @processor.ruleset_info[:loaded].to_f)
+                trace.set_tag('_dd.appsec.event_rules.error_count', @processor.ruleset_info[:failed].to_f)
+                trace.set_tag('_dd.appsec.event_rules.errors', JSON.dump(@processor.ruleset_info[:errors]))
+                trace.set_tag('_dd.appsec.event_rules.addresses', JSON.dump(@processor.addresses))
 
                 # Ensure these tags reach the backend
-                active_trace.keep!
+                trace.keep!
               end
             end
           end
 
-          def add_waf_runtime_tags(context)
-            return unless active_trace
+          def add_waf_runtime_tags(trace, context)
+            return unless trace
             return unless context
 
-            active_trace.set_tag('_dd.appsec.waf.timeouts', context.timeouts)
+            trace.set_tag('_dd.appsec.waf.timeouts', context.timeouts)
 
             # these tags expect time in us
-            active_trace.set_tag('_dd.appsec.waf.duration', context.time_ns / 1000.0)
-            active_trace.set_tag('_dd.appsec.waf.duration_ext', context.time_ext_ns / 1000.0)
+            trace.set_tag('_dd.appsec.waf.duration', context.time_ns / 1000.0)
+            trace.set_tag('_dd.appsec.waf.duration_ext', context.time_ext_ns / 1000.0)
           end
         end
       end

data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb

@@ -13,7 +13,7 @@ module Datadog
           # Watcher for Rails gateway events
           module Watcher
             def self.watch
-              Instrumentation.gateway.watch('rails.request.action') do |stack, request|
+              Instrumentation.gateway.watch('rails.request.action', :appsec) do |stack, request|
                 block = false
                 event = nil
                 waf_context = request.env['datadog.waf.context']
@@ -22,23 +22,24 @@ module Datadog
                   trace = active_trace
                   span = active_span
 
-                  Rails::Reactive::Action.subscribe(op, waf_context) do |action, result, _block|
-                    record = [:block, :monitor].include?(action)
-                    if record
+                  Rails::Reactive::Action.subscribe(op, waf_context) do |result, _block|
+                    if result.status == :match
                       # TODO: should this hash be an Event instance instead?
                       event = {
                         waf_result: result,
                         trace: trace,
                         span: span,
                         request: request,
-                        action: action
+                        actions: result.actions
                       }
 
+                      span.set_tag('appsec.event', 'true') if span
+
                       waf_context.events << event
                     end
                   end
 
-                  _action, _result, block = Rails::Reactive::Action.publish(op, request)
+                  _result, block = Rails::Reactive::Action.publish(op, request)
                 end
 
                 next [nil, [[:block, event]]] if block

data/lib/datadog/appsec/contrib/rails/reactive/action.rb

@@ -36,27 +36,27 @@ module Datadog
                 }
 
                 waf_timeout = Datadog::AppSec.settings.waf_timeout
-                action, result = waf_context.run(waf_args, waf_timeout)
+                result = waf_context.run(waf_args, waf_timeout)
 
                 Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
 
-                # TODO: encapsulate return array in a type
-                case action
-                when :monitor
+                case result.status
+                when :match
                   Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, false]
-                when :block
-                  Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, true]
-                  throw(:block, [action, result, true])
-                when :good
+
+                  block = result.actions.include?('block')
+
+                  yield [result, block]
+
+                  throw(:block, [result, true]) if block
+                when :ok
                   Datadog.logger.debug { "WAF OK: #{result.inspect}" }
                 when :invalid_call
                   Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
                 when :invalid_rule, :invalid_flow, :no_rule
                   Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
                 else
-                  Datadog.logger.debug { "WAF UNKNOWN: #{action.inspect} #{result.inspect}" }
+                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
                 end
               end
             end

data/lib/datadog/appsec/contrib/rails/request.rb

@@ -7,6 +7,9 @@ module Datadog
         # Normalized extration of data from ActionDispatch::Request
         module Request
           def self.parsed_body(request)
+            # force body parameter parsing, which is done lazily by Rails
+            request.parameters
+
             # usually Hash<String,String> but can be a more complex
             # Hash<String,String||Array||Hash> when e.g coming from JSON or
             # with Rails advanced param square bracket parsing

data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb

@@ -15,7 +15,7 @@ module Datadog
           module Watcher
             # rubocop:disable Metrics/MethodLength
             def self.watch
-              Instrumentation.gateway.watch('sinatra.request.dispatch') do |stack, request|
+              Instrumentation.gateway.watch('sinatra.request.dispatch', :appsec) do |stack, request|
                 block = false
                 event = nil
                 waf_context = request.env['datadog.waf.context']
@@ -24,23 +24,24 @@ module Datadog
                   trace = active_trace
                   span = active_span
 
-                  Rack::Reactive::RequestBody.subscribe(op, waf_context) do |action, result, _block|
-                    record = [:block, :monitor].include?(action)
-                    if record
+                  Rack::Reactive::RequestBody.subscribe(op, waf_context) do |result, _block|
+                    if result.status == :match
                       # TODO: should this hash be an Event instance instead?
                       event = {
                         waf_result: result,
                         trace: trace,
                         span: span,
                         request: request,
-                        action: action
+                        actions: result.actions
                       }
 
+                      span.set_tag('appsec.event', 'true') if span
+
                       waf_context.events << event
                     end
                   end
 
-                  _action, _result, block = Rack::Reactive::RequestBody.publish(op, request)
+                  _result, block = Rack::Reactive::RequestBody.publish(op, request)
                 end
 
                 next [nil, [[:block, event]]] if block
@@ -55,7 +56,7 @@ module Datadog
                 [ret, res]
               end
 
-              Instrumentation.gateway.watch('sinatra.request.routed') do |stack, (request, route_params)|
+              Instrumentation.gateway.watch('sinatra.request.routed', :appsec) do |stack, (request, route_params)|
                 block = false
                 event = nil
                 waf_context = request.env['datadog.waf.context']
@@ -64,23 +65,24 @@ module Datadog
                   trace = active_trace
                   span = active_span
 
-                  Sinatra::Reactive::Routed.subscribe(op, waf_context) do |action, result, _block|
-                    record = [:block, :monitor].include?(action)
-                    if record
+                  Sinatra::Reactive::Routed.subscribe(op, waf_context) do |result, _block|
+                    if result.status == :match
                       # TODO: should this hash be an Event instance instead?
                       event = {
                         waf_result: result,
                         trace: trace,
                         span: span,
                         request: request,
-                        action: action
+                        actions: result.actions
                       }
 
+                      span.set_tag('appsec.event', 'true') if span
+
                       waf_context.events << event
                     end
                   end
 
-                  _action, _result, block = Sinatra::Reactive::Routed.publish(op, [request, route_params])
+                  _result, block = Sinatra::Reactive::Routed.publish(op, [request, route_params])
                 end
 
                 next [nil, [[:block, event]]] if block

data/lib/datadog/appsec/contrib/sinatra/reactive/routed.rb

@@ -31,27 +31,27 @@ module Datadog
                 }
 
                 waf_timeout = Datadog::AppSec.settings.waf_timeout
-                action, result = waf_context.run(waf_args, waf_timeout)
+                result = waf_context.run(waf_args, waf_timeout)
 
                 Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
 
-                # TODO: encapsulate return array in a type
-                case action
-                when :monitor
+                case result.status
+                when :match
                   Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, false]
-                when :block
-                  Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, true]
-                  throw(:block, [action, result, true])
-                when :good
+
+                  block = result.actions.include?('block')
+
+                  yield [result, block]
+
+                  throw(:block, [result, true]) if block
+                when :ok
                   Datadog.logger.debug { "WAF OK: #{result.inspect}" }
                 when :invalid_call
                   Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
                 when :invalid_rule, :invalid_flow, :no_rule
                   Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
                 else
-                  Datadog.logger.debug { "WAF UNKNOWN: #{action.inspect} #{result.inspect}" }
+                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
                 end
               end
             end

data/lib/datadog/appsec/event.rb

@@ -51,9 +51,7 @@ module Datadog
         end
       end
 
-      # rubocop:disable Metrics/AbcSize
-      # rubocop:disable Metrics/MethodLength
-      def self.record_via_span(*events)
+      def self.record_via_span(*events) # rubocop:disable Metrics/AbcSize
         events.group_by { |e| e[:trace] }.each do |trace, event_group|
           unless trace
             Datadog.logger.debug { "{ error: 'no trace: cannot record', event_group: #{event_group.inspect}}" }
@@ -64,10 +62,6 @@ module Datadog
 
           # prepare and gather tags to apply
           trace_tags = event_group.each_with_object({}) do |event, tags|
-            span = event[:span]
-
-            span.set_tag('appsec.event', 'true') if span
-
             # TODO: assume HTTP request context for now
 
             if (request = event[:request])
@@ -81,9 +75,7 @@ module Datadog
 
               tags['http.host'] = request.host
               tags['http.useragent'] = request.user_agent
-              tags['network.client.ip'] = request.ip
-
-              # tags['actor.ip'] = request.ip # TODO: uses client IP resolution algorithm
+              tags['network.client.ip'] = request.env['REMOTE_ADDR'] if request.env['REMOTE_ADDR']
             end
 
             if (response = event[:response])
@@ -114,8 +106,6 @@ module Datadog
           end
         end
       end
-      # rubocop:enable Metrics/MethodLength
-      # rubocop:enable Metrics/AbcSize
     end
   end
 end

data/lib/datadog/appsec/instrumentation/gateway.rb

@@ -6,6 +6,20 @@ module Datadog
     module Instrumentation
       # Instrumentation gateway implementation
       class Gateway
+        # Instrumentation gateway middleware
+        class Middleware
+          attr_reader :key, :block
+
+          def initialize(key, &block)
+            @key = key
+            @block = block
+          end
+
+          def call(*args, **kwargs, &block)
+            @block.call(*args, **kwargs, &block)
+          end
+        end
+
         def initialize
           @middlewares = Hash.new { |h, k| h[k] = [] }
         end
@@ -31,8 +45,8 @@ module Datadog
           stack.call(env)
         end
 
-        def watch(name, &block)
-          @middlewares[name] << block
+        def watch(name, key, &block)
+          @middlewares[name] << Middleware.new(key, &block) unless @middlewares[name].any? { |m| m.key == key }
         end
       end