ddtrace 1.0.0.beta1 → 1.1.0

data/lib/datadog/appsec/contrib/rails/patcher.rb

@@ -3,9 +3,10 @@
 require 'datadog/core/utils/only_once'
 
 require 'datadog/appsec/contrib/patcher'
-require 'datadog/appsec/contrib/rails/integration'
 require 'datadog/appsec/contrib/rails/framework'
 require 'datadog/appsec/contrib/rack/request_middleware'
+require 'datadog/appsec/contrib/rack/request_body_middleware'
+require 'datadog/appsec/contrib/rails/gateway/watcher'
 
 require 'datadog/tracing/contrib/rack/middlewares'
 
@@ -31,6 +32,7 @@ module Datadog
           end
 
           def patch
+            Gateway::Watcher.watch
             patch_before_intialize
             patch_after_intialize
 
@@ -48,7 +50,8 @@ module Datadog
               # Middleware must be added before the application is initialized.
               # Otherwise the middleware stack will be frozen.
               # Sometimes we don't want to activate middleware e.g. OpenTracing, etc.
-              add_middleware(app) if Datadog.configuration[:rails][:middleware]
+              add_middleware(app) if Datadog.configuration.tracing[:rails][:middleware]
+              patch_process_action
             end
           end
 
@@ -62,6 +65,36 @@ module Datadog
             end
           end
 
+          # Hook into ActionController::Instrumentation#process_action, which encompasses action filters
+          module ProcessActionPatch
+            def process_action(*args)
+              env = request.env
+
+              context = env['datadog.waf.context']
+
+              return super unless context
+
+              # TODO: handle exceptions, except for super
+
+              request_return, request_response = Instrumentation.gateway.push('rails.request.action', request) do
+                super
+              end
+
+              if request_response && request_response.any? { |action, _event| action == :block }
+                @_response = ::ActionDispatch::Response.new(403,
+                                                            { 'Content-Type' => 'text/html' },
+                                                            [Datadog::AppSec::Assets.blocked])
+                request_return = @_response.body
+              end
+
+              request_return
+            end
+          end
+
+          def patch_process_action
+            ActionController::Instrumentation.prepend(ProcessActionPatch)
+          end
+
           def include_middleware?(middleware, app)
             found = false
 

data/lib/datadog/appsec/contrib/rails/reactive/action.rb

@@ -0,0 +1,68 @@
+# typed: true
+
+require 'datadog/appsec/contrib/rails/request'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rails
+        module Reactive
+          # Dispatch data from a Rails request to the WAF context
+          module Action
+            def self.publish(op, request)
+              catch(:block) do
+                # params have been parsed from the request body
+                op.publish('rails.request.body', Rails::Request.parsed_body(request))
+                op.publish('rails.request.route_params', Rails::Request.route_params(request))
+
+                nil
+              end
+            end
+
+            def self.subscribe(op, waf_context)
+              addresses = [
+                'rails.request.body',
+                'rails.request.route_params',
+              ]
+
+              op.subscribe(*addresses) do |*values|
+                Datadog.logger.debug { "reacted to #{addresses.inspect}: #{values.inspect}" }
+                body = values[0]
+                path_params = values[1]
+
+                waf_args = {
+                  'server.request.body' => body,
+                  'server.request.path_params' => path_params,
+                }
+
+                waf_timeout = Datadog::AppSec.settings.waf_timeout
+                action, result = waf_context.run(waf_args, waf_timeout)
+
+                Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
+
+                # TODO: encapsulate return array in a type
+                case action
+                when :monitor
+                  Datadog.logger.debug { "WAF: #{result.inspect}" }
+                  yield [action, result, false]
+                when :block
+                  Datadog.logger.debug { "WAF: #{result.inspect}" }
+                  yield [action, result, true]
+                  throw(:block, [action, result, true])
+                when :good
+                  Datadog.logger.debug { "WAF OK: #{result.inspect}" }
+                when :invalid_call
+                  Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
+                when :invalid_rule, :invalid_flow, :no_rule
+                  Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
+                else
+                  Datadog.logger.debug { "WAF UNKNOWN: #{action.inspect} #{result.inspect}" }
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rails/request.rb

@@ -0,0 +1,33 @@
+# typed: true
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rails
+        # Normalized extration of data from ActionDispatch::Request
+        module Request
+          def self.parsed_body(request)
+            # usually Hash<String,String> but can be a more complex
+            # Hash<String,String||Array||Hash> when e.g coming from JSON or
+            # with Rails advanced param square bracket parsing
+            body = request.env['action_dispatch.request.request_parameters']
+
+            return if body.nil?
+
+            body.reject do |k, _v|
+              request.env['action_dispatch.request.path_parameters'].key?(k)
+            end
+          end
+
+          def self.route_params(request)
+            excluded = [:controller, :action]
+
+            request.env['action_dispatch.request.path_parameters'].reject do |k, _v|
+              excluded.include?(k)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rails/request_middleware.rb

@@ -1,3 +1,5 @@
+# typed: true
+
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/contrib/sinatra/configuration/settings.rb

@@ -1,3 +1,5 @@
+# typed: false
+
 require 'datadog/appsec/contrib/configuration/settings'
 require 'datadog/appsec/contrib/sinatra/ext'
 

data/lib/datadog/appsec/contrib/sinatra/ext.rb

@@ -1,3 +1,5 @@
+# typed: true
+
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/contrib/sinatra/framework.rb

@@ -1,3 +1,5 @@
+# typed: true
+
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb

@@ -0,0 +1,124 @@
+# typed: false
+
+require 'datadog/appsec/instrumentation/gateway'
+require 'datadog/appsec/reactive/operation'
+require 'datadog/appsec/contrib/rack/reactive/request_body'
+require 'datadog/appsec/contrib/sinatra/reactive/routed'
+require 'datadog/appsec/event'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Sinatra
+        module Gateway
+          # Watcher for Rails gateway events
+          module Watcher
+            # rubocop:disable Metrics/MethodLength
+            def self.watch
+              Instrumentation.gateway.watch('sinatra.request.dispatch') do |stack, request|
+                block = false
+                event = nil
+                waf_context = request.env['datadog.waf.context']
+
+                AppSec::Reactive::Operation.new('sinatra.request.dispatch') do |op|
+                  trace = active_trace
+                  span = active_span
+
+                  Rack::Reactive::RequestBody.subscribe(op, waf_context) do |action, result, _block|
+                    record = [:block, :monitor].include?(action)
+                    if record
+                      # TODO: should this hash be an Event instance instead?
+                      event = {
+                        waf_result: result,
+                        trace: trace,
+                        span: span,
+                        request: request,
+                        action: action
+                      }
+
+                      waf_context.events << event
+                    end
+                  end
+
+                  _action, _result, block = Rack::Reactive::RequestBody.publish(op, request)
+                end
+
+                next [nil, [[:block, event]]] if block
+
+                ret, res = stack.call(request)
+
+                if event
+                  res ||= []
+                  res << [:monitor, event]
+                end
+
+                [ret, res]
+              end
+
+              Instrumentation.gateway.watch('sinatra.request.routed') do |stack, (request, route_params)|
+                block = false
+                event = nil
+                waf_context = request.env['datadog.waf.context']
+
+                AppSec::Reactive::Operation.new('sinatra.request.routed') do |op|
+                  trace = active_trace
+                  span = active_span
+
+                  Sinatra::Reactive::Routed.subscribe(op, waf_context) do |action, result, _block|
+                    record = [:block, :monitor].include?(action)
+                    if record
+                      # TODO: should this hash be an Event instance instead?
+                      event = {
+                        waf_result: result,
+                        trace: trace,
+                        span: span,
+                        request: request,
+                        action: action
+                      }
+
+                      waf_context.events << event
+                    end
+                  end
+
+                  _action, _result, block = Sinatra::Reactive::Routed.publish(op, [request, route_params])
+                end
+
+                next [nil, [[:block, event]]] if block
+
+                ret, res = stack.call(request)
+
+                if event
+                  res ||= []
+                  res << [:monitor, event]
+                end
+
+                [ret, res]
+              end
+            end
+            # rubocop:enable Metrics/MethodLength
+
+            class << self
+              private
+
+              def active_trace
+                # TODO: factor out tracing availability detection
+
+                return unless defined?(Datadog::Tracing)
+
+                Datadog::Tracing.active_trace
+              end
+
+              def active_span
+                # TODO: factor out tracing availability detection
+
+                return unless defined?(Datadog::Tracing)
+
+                Datadog::Tracing.active_span
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/sinatra/patcher.rb

@@ -3,9 +3,9 @@
 require 'datadog/tracing/contrib/rack/middlewares'
 
 require 'datadog/appsec/contrib/patcher'
-require 'datadog/appsec/contrib/sinatra/integration'
 require 'datadog/appsec/contrib/rack/request_middleware'
 require 'datadog/appsec/contrib/sinatra/framework'
+require 'datadog/appsec/contrib/sinatra/gateway/watcher'
 require 'datadog/tracing/contrib/sinatra/framework'
 
 module Datadog
@@ -37,11 +37,68 @@ module Datadog
               else
                 tracing_sinatra_framework.add_middleware(Datadog::AppSec::Contrib::Rack::RequestMiddleware, builder)
               end
+
               tracing_sinatra_framework.inspect_middlewares(builder)
             end
           end
         end
 
+        # Hook into Base#dispatch!, which encompasses route filters
+        module DispatchPatch
+          def dispatch!
+            env = @request.env
+
+            context = env['datadog.waf.context']
+
+            return super unless context
+
+            # TODO: handle exceptions, except for super
+
+            request_return, request_response = Instrumentation.gateway.push('sinatra.request.dispatch', request) do
+              super
+            end
+
+            if request_response && request_response.any? { |action, _event| action == :block }
+              self.response = ::Sinatra::Response.new([Datadog::AppSec::Assets.blocked],
+                                                      403,
+                                                      { 'Content-Type' => 'text/html' })
+              request_return = nil
+            end
+
+            request_return
+          end
+        end
+
+        # Hook into Base#route_eval, which
+        # path params are returned by pattern.params in process_route, then
+        # merged with normal params, so we get both
+        module RoutePatch
+          def process_route(*)
+            env = @request.env
+
+            context = env['datadog.waf.context']
+
+            return super unless context
+
+            # process_route is called repeatedly until a route is found.
+            # Until then, params has no route params.
+            # Capture normal params.
+            base_params = params
+
+            super do |*args|
+              # This block is called only once the route is found.
+              # At this point params has both route params and normal params.
+              route_params = params.each.with_object({}) { |(k, v), h| h[k] = v unless base_params.key?(k) }
+
+              Instrumentation.gateway.push('sinatra.request.routed', [request, route_params])
+
+              # TODO: handle block
+
+              yield(*args)
+            end
+          end
+        end
+
         # Patcher for AppSec on Sinatra
         module Patcher
           include Datadog::AppSec::Contrib::Patcher
@@ -57,9 +114,11 @@ module Datadog
           end
 
           def patch
+            Gateway::Watcher.watch
             patch_default_middlewares
+            patch_dispatch
+            patch_route
             setup_security
-
             Patcher.instance_variable_set(:@patched, true)
           end
 
@@ -70,6 +129,14 @@ module Datadog
           def patch_default_middlewares
             ::Sinatra::Base.singleton_class.prepend(DefaultMiddlewarePatch)
           end
+
+          def patch_dispatch
+            ::Sinatra::Base.prepend(DispatchPatch)
+          end
+
+          def patch_route
+            ::Sinatra::Base.prepend(RoutePatch)
+          end
         end
       end
     end

data/lib/datadog/appsec/contrib/sinatra/reactive/routed.rb

@@ -0,0 +1,63 @@
+# typed: true
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Sinatra
+        module Reactive
+          # Dispatch data from a Rack request to the WAF context
+          module Routed
+            def self.publish(op, data)
+              _request, route_params = data
+
+              catch(:block) do
+                op.publish('sinatra.request.route_params', route_params)
+
+                nil
+              end
+            end
+
+            def self.subscribe(op, waf_context)
+              addresses = [
+                'sinatra.request.route_params',
+              ]
+
+              op.subscribe(*addresses) do |*values|
+                Datadog.logger.debug { "reacted to #{addresses.inspect}: #{values.inspect}" }
+                path_params = values[0]
+
+                waf_args = {
+                  'server.request.path_params' => path_params,
+                }
+
+                waf_timeout = Datadog::AppSec.settings.waf_timeout
+                action, result = waf_context.run(waf_args, waf_timeout)
+
+                Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
+
+                # TODO: encapsulate return array in a type
+                case action
+                when :monitor
+                  Datadog.logger.debug { "WAF: #{result.inspect}" }
+                  yield [action, result, false]
+                when :block
+                  Datadog.logger.debug { "WAF: #{result.inspect}" }
+                  yield [action, result, true]
+                  throw(:block, [action, result, true])
+                when :good
+                  Datadog.logger.debug { "WAF OK: #{result.inspect}" }
+                when :invalid_call
+                  Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
+                when :invalid_rule, :invalid_flow, :no_rule
+                  Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
+                else
+                  Datadog.logger.debug { "WAF UNKNOWN: #{action.inspect} #{result.inspect}" }
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/sinatra/request_middleware.rb

@@ -1,3 +1,5 @@
+# typed: true
+
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/event.rb

@@ -1,3 +1,7 @@
+# typed: false
+
+require 'json'
+
 require 'datadog/appsec/contrib/rack/request'
 require 'datadog/appsec/contrib/rack/response'
 require 'datadog/appsec/rate_limiter'
@@ -34,7 +38,14 @@ module Datadog
         Content-Language
       ].map!(&:downcase).freeze
 
+      # Record events for a trace
+      #
+      # This is expected to be called only once per trace for the rate limiter
+      # to properly apply
       def self.record(*events)
+        # ensure rate limiter is called only when there are events to record
+        return if events.empty?
+
         Datadog::AppSec::RateLimiter.limit(:traces) do
           record_via_span(*events)
         end
@@ -43,49 +54,48 @@ module Datadog
       # rubocop:disable Metrics/AbcSize
       # rubocop:disable Metrics/MethodLength
       def self.record_via_span(*events)
-        events.group_by { |e| e[:root_span] }.each do |root_span, event_group|
-          unless root_span
-            Datadog.logger.debug { "{ error: 'no root span: cannot record', event_group: #{event_group.inspect}}" }
+        events.group_by { |e| e[:trace] }.each do |trace, event_group|
+          unless trace
+            Datadog.logger.debug { "{ error: 'no trace: cannot record', event_group: #{event_group.inspect}}" }
             next
           end
 
-          # TODO: this is a hack but there is no API to do that
-          root_span_tags = root_span.send(:meta).keys
+          trace.keep!
 
           # prepare and gather tags to apply
-          root_tags = event_group.each_with_object({}) do |event, tags|
+          trace_tags = event_group.each_with_object({}) do |event, tags|
             span = event[:span]
-            trace = event[:trace]
 
-            if span
-              span.set_tag('appsec.event', 'true')
-              trace.keep!
-            end
-
-            request = event[:request]
-            response = event[:response]
+            span.set_tag('appsec.event', 'true') if span
 
             # TODO: assume HTTP request context for now
-            request_headers = AppSec::Contrib::Rack::Request.headers(request).select do |k, _|
-              ALLOWED_REQUEST_HEADERS.include?(k.downcase)
-            end
-            response_headers = AppSec::Contrib::Rack::Response.headers(response).select do |k, _|
-              ALLOWED_RESPONSE_HEADERS.include?(k.downcase)
-            end
 
-            request_headers.each do |header, value|
-              tags["http.request.headers.#{header}"] = value
-            end
+            if (request = event[:request])
+              request_headers = AppSec::Contrib::Rack::Request.headers(request).select do |k, _|
+                ALLOWED_REQUEST_HEADERS.include?(k.downcase)
+              end
+
+              request_headers.each do |header, value|
+                tags["http.request.headers.#{header}"] = value
+              end
 
-            response_headers.each do |header, value|
-              tags["http.response.headers.#{header}"] = value
+              tags['http.host'] = request.host
+              tags['http.useragent'] = request.user_agent
+              tags['network.client.ip'] = request.ip
+
+              # tags['actor.ip'] = request.ip # TODO: uses client IP resolution algorithm
             end
 
-            tags['http.host'] = request.host
-            tags['http.useragent'] = request.user_agent
-            tags['network.client.ip'] = request.ip
+            if (response = event[:response])
+              response_headers = AppSec::Contrib::Rack::Response.headers(response).select do |k, _|
+                ALLOWED_RESPONSE_HEADERS.include?(k.downcase)
+              end
+
+              response_headers.each do |header, value|
+                tags["http.response.headers.#{header}"] = value
+              end
+            end
 
-            # tags['actor.ip'] = request.ip # TODO: uses client IP resolution algorithm
             tags['_dd.origin'] = 'appsec'
 
             # accumulate triggers
@@ -96,16 +106,16 @@ module Datadog
           # apply tags to root span
 
           # complex types are unsupported, we need to serialize to a string
-          triggers = root_tags.delete('_dd.appsec.triggers')
-          root_span.set_tag('_dd.appsec.json', JSON.dump({ triggers: triggers }))
+          triggers = trace_tags.delete('_dd.appsec.triggers')
+          trace.set_tag('_dd.appsec.json', JSON.dump({ triggers: triggers }))
 
-          root_tags.each do |key, value|
-            root_span.set_tag(key, value.is_a?(String) ? value.encode('UTf-8') : value) unless root_span_tags.include?(key)
+          trace_tags.each do |key, value|
+            trace.set_tag(key, value)
           end
         end
       end
-      # rubocop:enable Metrics/AbcSize
       # rubocop:enable Metrics/MethodLength
+      # rubocop:enable Metrics/AbcSize
     end
   end
 end

data/lib/datadog/appsec/extensions.rb

@@ -1,5 +1,5 @@
-# typed: true
-require 'forwardable'
+# typed: false
+
 require 'datadog/appsec/configuration'
 
 module Datadog
@@ -24,8 +24,6 @@ module Datadog
       # Merges {Datadog::AppSec::Configuration::Settings} and {Datadog::AppSec::Configuration::DSL}
       # into a single read/write object.
       class AppSecAdapter
-        extend Forwardable
-
         def initialize(settings)
           @settings = settings
         end

data/lib/datadog/appsec/instrumentation/gateway.rb

@@ -1,3 +1,5 @@
+# typed: true
+
 module Datadog
   module AppSec
     # Instrumentation for AppSec