ddtrace 1.0.0 → 1.9.0

data/lib/datadog/appsec/contrib/rack/reactive/request_body.rb

@@ -0,0 +1,64 @@
+# typed: ignore
+
+require_relative '../request'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rack
+        module Reactive
+          # Dispatch data from a Rack request to the WAF context
+          module RequestBody
+            def self.publish(op, request)
+              catch(:block) do
+                # params have been parsed from the request body
+                op.publish('request.body', Rack::Request.form_hash(request))
+
+                nil
+              end
+            end
+
+            def self.subscribe(op, waf_context)
+              addresses = [
+                'request.body',
+              ]
+
+              op.subscribe(*addresses) do |*values|
+                Datadog.logger.debug { "reacted to #{addresses.inspect}: #{values.inspect}" }
+                body = values[0]
+
+                waf_args = {
+                  'server.request.body' => body,
+                }
+
+                waf_timeout = Datadog::AppSec.settings.waf_timeout
+                result = waf_context.run(waf_args, waf_timeout)
+
+                Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
+
+                case result.status
+                when :match
+                  Datadog.logger.debug { "WAF: #{result.inspect}" }
+
+                  block = result.actions.include?('block')
+
+                  yield [result, block]
+
+                  throw(:block, [result, true]) if block
+                when :ok
+                  Datadog.logger.debug { "WAF OK: #{result.inspect}" }
+                when :invalid_call
+                  Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
+                when :invalid_rule, :invalid_flow, :no_rule
+                  Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
+                else
+                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rack/reactive/response.rb

@@ -1,6 +1,6 @@
-# typed: true
+# typed: ignore
 
-require 'datadog/appsec/contrib/rack/response'
+require_relative '../response'
 
 module Datadog
   module AppSec
@@ -32,27 +32,27 @@ module Datadog
                 }
 
                 waf_timeout = Datadog::AppSec.settings.waf_timeout
-                action, result = waf_context.run(waf_args, waf_timeout)
+                result = waf_context.run(waf_args, waf_timeout)
 
                 Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
 
-                # TODO: encapsulate return array in a type
-                case action
-                when :monitor
+                case result.status
+                when :match
                   Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, false]
-                when :block
-                  Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, true]
-                  throw(:block, [action, result, true])
-                when :good
+
+                  block = result.actions.include?('block')
+
+                  yield [result, block]
+
+                  throw(:block, [result, true]) if block
+                when :ok
                   Datadog.logger.debug { "WAF OK: #{result.inspect}" }
                 when :invalid_call
                   Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
                 when :invalid_rule, :invalid_flow, :no_rule
                   Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
                 else
-                  Datadog.logger.debug { "WAF UNKNOWN: #{action.inspect} #{result.inspect}" }
+                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
                 end
               end
             end

data/lib/datadog/appsec/contrib/rack/request.rb

@@ -1,4 +1,7 @@
-# typed: true
+# typed: ignore
+
+require_relative '../../../tracing/client_ip'
+require_relative '../../../tracing/contrib/rack/header_collection'
 
 module Datadog
   module AppSec
@@ -45,6 +48,29 @@ module Datadog
           def self.cookies(request)
             request.cookies
           end
+
+          def self.form_hash(request)
+            # force form data processing
+            request.POST if request.form_data?
+
+            # usually Hash<String,String> but can be a more complex
+            # Hash<String,String||Array||Hash> when e.g coming from JSON
+            request.env['rack.request.form_hash']
+          end
+
+          def self.client_ip(request)
+            remote_ip = request.env['REMOTE_ADDR']
+            headers = Datadog::Tracing::Contrib::Rack::Header::RequestHeaderCollection.new(request.env)
+
+            result = Datadog::Tracing::ClientIp.raw_ip_from_request(headers, remote_ip)
+
+            if result.raw_ip
+              ip = Datadog::Tracing::ClientIp.strip_decorations(result.raw_ip)
+              return unless Datadog::Tracing::ClientIp.valid_ip?(ip)
+
+              ip
+            end
+          end
         end
       end
     end

data/lib/datadog/appsec/contrib/rack/request_body_middleware.rb

@@ -0,0 +1,41 @@
+# typed: ignore
+
+require_relative '../../instrumentation/gateway'
+require_relative '../../response'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rack
+        # Rack request body middleware for AppSec
+        # This should be inserted just below Rack::JSONBodyParser or
+        # legacy Rack::PostBodyContentTypeParser from rack-contrib
+        class RequestBodyMiddleware
+          def initialize(app, opt = {})
+            @app = app
+          end
+
+          def call(env)
+            context = env['datadog.waf.context']
+
+            return @app.call(env) unless context
+
+            # TODO: handle exceptions, except for @app.call
+
+            request = ::Rack::Request.new(env)
+
+            request_return, request_response = Instrumentation.gateway.push('rack.request.body', request) do
+              @app.call(env)
+            end
+
+            if request_response && request_response.any? { |action, _event| action == :block }
+              request_return = AppSec::Response.negotiate(env).to_rack
+            end
+
+            request_return
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -1,8 +1,13 @@
 # typed: ignore
 
-require 'datadog/appsec/instrumentation/gateway'
-require 'datadog/appsec/processor'
-require 'datadog/appsec/assets'
+require 'json'
+
+require_relative '../../instrumentation/gateway'
+require_relative '../../processor'
+require_relative '../../response'
+
+require_relative '../../../tracing/client_ip'
+require_relative '../../../tracing/contrib/rack/header_collection'
 
 module Datadog
   module AppSec
@@ -14,11 +19,12 @@ module Datadog
           def initialize(app, opt = {})
             @app = app
 
+            @oneshot_tags_sent = false
             @processor = Datadog::AppSec::Processor.new
           end
 
           def call(env)
-            return @app.call(env) unless @processor.ready?
+            return @app.call(env) unless Datadog.configuration.appsec.enabled && @processor.ready?
 
             # TODO: handle exceptions, except for @app.call
 
@@ -27,12 +33,14 @@ module Datadog
             env['datadog.waf.context'] = context
             request = ::Rack::Request.new(env)
 
+            add_appsec_tags(active_trace, active_span, env)
+
             request_return, request_response = Instrumentation.gateway.push('rack.request', request) do
               @app.call(env)
             end
 
             if request_response && request_response.any? { |action, _event| action == :block }
-              request_return = [403, { 'Content-Type' => 'text/html' }, [Datadog::AppSec::Assets.blocked]]
+              request_return = AppSec::Response.negotiate(env).to_rack
             end
 
             response = ::Rack::Response.new(request_return[2], request_return[0], request_return[1])
@@ -40,15 +48,89 @@ module Datadog
               @waf_context = context
             end
 
-            _, response_response = Instrumentation.gateway.push('rack.response', response)
+            _response_return, _response_response = Instrumentation.gateway.push('rack.response', response)
 
-            request_response.each { |_, e| e.merge!(response: response) } if request_response
-            response_response.each { |_, e| e.merge!(request: request) } if response_response
-            both_response = (request_response || []) + (response_response || [])
+            context.events.each do |e|
+              e[:response] ||= response
+              e[:request]  ||= request
+            end
 
-            AppSec::Event.record(*both_response.map { |_action, event| event }) if both_response.any?
+            AppSec::Event.record(*context.events)
 
             request_return
+          ensure
+            add_waf_runtime_tags(active_trace, context) if context
+            context.finalize if context
+          end
+
+          private
+
+          def active_trace
+            # TODO: factor out tracing availability detection
+
+            return unless defined?(Datadog::Tracing)
+
+            Datadog::Tracing.active_trace
+          end
+
+          def active_span
+            # TODO: factor out tracing availability detection
+
+            return unless defined?(Datadog::Tracing)
+
+            Datadog::Tracing.active_span
+          end
+
+          def add_appsec_tags(trace, span, env)
+            return unless trace
+
+            trace.set_tag('_dd.appsec.enabled', 1)
+            trace.set_tag('_dd.runtime_family', 'ruby')
+            trace.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
+
+            if span && span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_CLIENT_IP).nil?
+              request_header_collection = Datadog::Tracing::Contrib::Rack::Header::RequestHeaderCollection.new(env)
+
+              # always collect client ip, as this is part of AppSec provided functionality
+              Datadog::Tracing::ClientIp.set_client_ip_tag!(
+                span,
+                headers: request_header_collection,
+                remote_ip: env['REMOTE_ADDR']
+              )
+            end
+
+            if @processor.ruleset_info
+              trace.set_tag('_dd.appsec.event_rules.version', @processor.ruleset_info[:version])
+
+              unless @oneshot_tags_sent
+                # Small race condition, but it's inoccuous: worst case the tags
+                # are sent a couple of times more than expected
+                @oneshot_tags_sent = true
+
+                trace.set_tag('_dd.appsec.event_rules.loaded', @processor.ruleset_info[:loaded].to_f)
+                trace.set_tag('_dd.appsec.event_rules.error_count', @processor.ruleset_info[:failed].to_f)
+                trace.set_tag('_dd.appsec.event_rules.errors', JSON.dump(@processor.ruleset_info[:errors]))
+                trace.set_tag('_dd.appsec.event_rules.addresses', JSON.dump(@processor.addresses))
+
+                # Ensure these tags reach the backend
+                trace.keep!
+                trace.set_tag(
+                  Datadog::Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
+                  Datadog::Tracing::Sampling::Ext::Decision::ASM
+                )
+              end
+            end
+          end
+
+          def add_waf_runtime_tags(trace, context)
+            return unless trace
+            return unless context
+
+            trace.set_tag('_dd.appsec.waf.timeouts', context.timeouts)
+
+            # these tags expect time in us
+            trace.set_tag('_dd.appsec.waf.duration', context.time_ns / 1000.0)
+            trace.set_tag('_dd.appsec.waf.duration_ext', context.time_ext_ns / 1000.0)
           end
         end
       end

data/lib/datadog/appsec/contrib/rack/response.rb

@@ -1,4 +1,4 @@
-# typed: true
+# typed: ignore
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/contrib/rails/configuration/settings.rb

@@ -1,7 +1,7 @@
-# typed: false
+# typed: ignore
 
-require 'datadog/appsec/contrib/configuration/settings'
-require 'datadog/appsec/contrib/rails/ext'
+require_relative '../../configuration/settings'
+require_relative '../ext'
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/contrib/rails/ext.rb

@@ -1,4 +1,4 @@
-# typed: true
+# typed: ignore
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/contrib/rails/framework.rb

@@ -1,4 +1,4 @@
-# typed: true
+# typed: ignore
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb

@@ -0,0 +1,82 @@
+# typed: ignore
+
+require_relative '../../../instrumentation/gateway'
+require_relative '../../../reactive/operation'
+require_relative '../reactive/action'
+require_relative '../../../event'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rails
+        module Gateway
+          # Watcher for Rails gateway events
+          module Watcher
+            def self.watch
+              Instrumentation.gateway.watch('rails.request.action', :appsec) do |stack, request|
+                block = false
+                event = nil
+                waf_context = request.env['datadog.waf.context']
+
+                AppSec::Reactive::Operation.new('rails.request.action') do |op|
+                  trace = active_trace
+                  span = active_span
+
+                  Rails::Reactive::Action.subscribe(op, waf_context) do |result, _block|
+                    if result.status == :match
+                      # TODO: should this hash be an Event instance instead?
+                      event = {
+                        waf_result: result,
+                        trace: trace,
+                        span: span,
+                        request: request,
+                        actions: result.actions
+                      }
+
+                      span.set_tag('appsec.event', 'true') if span
+
+                      waf_context.events << event
+                    end
+                  end
+
+                  _result, block = Rails::Reactive::Action.publish(op, request)
+                end
+
+                next [nil, [[:block, event]]] if block
+
+                ret, res = stack.call(request)
+
+                if event
+                  res ||= []
+                  res << [:monitor, event]
+                end
+
+                [ret, res]
+              end
+            end
+
+            class << self
+              private
+
+              def active_trace
+                # TODO: factor out tracing availability detection
+
+                return unless defined?(Datadog::Tracing)
+
+                Datadog::Tracing.active_trace
+              end
+
+              def active_span
+                # TODO: factor out tracing availability detection
+
+                return unless defined?(Datadog::Tracing)
+
+                Datadog::Tracing.active_span
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rails/integration.rb

@@ -1,10 +1,10 @@
 # typed: ignore
 
-require 'datadog/appsec/contrib/integration'
+require_relative '../integration'
 
-require 'datadog/appsec/contrib/rails/configuration/settings'
-require 'datadog/appsec/contrib/rails/patcher'
-require 'datadog/appsec/contrib/rails/request_middleware'
+require_relative 'configuration/settings'
+require_relative 'patcher'
+require_relative 'request_middleware'
 
 module Datadog
   module AppSec
@@ -19,7 +19,7 @@ module Datadog
           register_as :rails, auto_patch: false
 
           def self.version
-            Gem.loaded_specs['rails'] && Gem.loaded_specs['rails'].version
+            Gem.loaded_specs['railties'] && Gem.loaded_specs['railties'].version
           end
 
           def self.loaded?

data/lib/datadog/appsec/contrib/rails/patcher.rb

@@ -1,13 +1,15 @@
 # typed: ignore
 
-require 'datadog/core/utils/only_once'
+require_relative '../../../core/utils/only_once'
 
-require 'datadog/appsec/contrib/patcher'
-require 'datadog/appsec/contrib/rails/integration'
-require 'datadog/appsec/contrib/rails/framework'
-require 'datadog/appsec/contrib/rack/request_middleware'
+require_relative '../patcher'
+require_relative 'framework'
+require_relative '../../response'
+require_relative '../rack/request_middleware'
+require_relative '../rack/request_body_middleware'
+require_relative 'gateway/watcher'
 
-require 'datadog/tracing/contrib/rack/middlewares'
+require_relative '../../../tracing/contrib/rack/middlewares'
 
 module Datadog
   module AppSec
@@ -31,6 +33,7 @@ module Datadog
           end
 
           def patch
+            Gateway::Watcher.watch
             patch_before_intialize
             patch_after_intialize
 
@@ -49,19 +52,50 @@ module Datadog
               # Otherwise the middleware stack will be frozen.
               # Sometimes we don't want to activate middleware e.g. OpenTracing, etc.
               add_middleware(app) if Datadog.configuration.tracing[:rails][:middleware]
+              patch_process_action
             end
           end
 
           def add_middleware(app)
             # Add trace middleware
             if include_middleware?(Datadog::Tracing::Contrib::Rack::TraceMiddleware, app)
-              app.middleware.insert_after(Datadog::Tracing::Contrib::Rack::TraceMiddleware,
-                                          Datadog::AppSec::Contrib::Rack::RequestMiddleware)
+              app.middleware.insert_after(
+                Datadog::Tracing::Contrib::Rack::TraceMiddleware,
+                Datadog::AppSec::Contrib::Rack::RequestMiddleware
+              )
             else
               app.middleware.insert_before(0, Datadog::AppSec::Contrib::Rack::RequestMiddleware)
             end
           end
 
+          # Hook into ActionController::Instrumentation#process_action, which encompasses action filters
+          module ProcessActionPatch
+            def process_action(*args)
+              env = request.env
+
+              context = env['datadog.waf.context']
+
+              return super unless context
+
+              # TODO: handle exceptions, except for super
+
+              request_return, request_response = Instrumentation.gateway.push('rails.request.action', request) do
+                super
+              end
+
+              if request_response && request_response.any? { |action, _event| action == :block }
+                @_response = AppSec::Response.negotiate(env).to_action_dispatch_response
+                request_return = @_response.body
+              end
+
+              request_return
+            end
+          end
+
+          def patch_process_action
+            ::ActionController::Metal.prepend(ProcessActionPatch)
+          end
+
           def include_middleware?(middleware, app)
             found = false
 

data/lib/datadog/appsec/contrib/rails/reactive/action.rb

@@ -0,0 +1,68 @@
+# typed: ignore
+
+require_relative '../request'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rails
+        module Reactive
+          # Dispatch data from a Rails request to the WAF context
+          module Action
+            def self.publish(op, request)
+              catch(:block) do
+                # params have been parsed from the request body
+                op.publish('rails.request.body', Rails::Request.parsed_body(request))
+                op.publish('rails.request.route_params', Rails::Request.route_params(request))
+
+                nil
+              end
+            end
+
+            def self.subscribe(op, waf_context)
+              addresses = [
+                'rails.request.body',
+                'rails.request.route_params',
+              ]
+
+              op.subscribe(*addresses) do |*values|
+                Datadog.logger.debug { "reacted to #{addresses.inspect}: #{values.inspect}" }
+                body = values[0]
+                path_params = values[1]
+
+                waf_args = {
+                  'server.request.body' => body,
+                  'server.request.path_params' => path_params,
+                }
+
+                waf_timeout = Datadog::AppSec.settings.waf_timeout
+                result = waf_context.run(waf_args, waf_timeout)
+
+                Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
+
+                case result.status
+                when :match
+                  Datadog.logger.debug { "WAF: #{result.inspect}" }
+
+                  block = result.actions.include?('block')
+
+                  yield [result, block]
+
+                  throw(:block, [result, true]) if block
+                when :ok
+                  Datadog.logger.debug { "WAF OK: #{result.inspect}" }
+                when :invalid_call
+                  Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
+                when :invalid_rule, :invalid_flow, :no_rule
+                  Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
+                else
+                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rails/request.rb

@@ -0,0 +1,36 @@
+# typed: ignore
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rails
+        # Normalized extration of data from ActionDispatch::Request
+        module Request
+          def self.parsed_body(request)
+            # force body parameter parsing, which is done lazily by Rails
+            request.parameters
+
+            # usually Hash<String,String> but can be a more complex
+            # Hash<String,String||Array||Hash> when e.g coming from JSON or
+            # with Rails advanced param square bracket parsing
+            body = request.env['action_dispatch.request.request_parameters']
+
+            return if body.nil?
+
+            body.reject do |k, _v|
+              request.env['action_dispatch.request.path_parameters'].key?(k)
+            end
+          end
+
+          def self.route_params(request)
+            excluded = [:controller, :action]
+
+            request.env['action_dispatch.request.path_parameters'].reject do |k, _v|
+              excluded.include?(k)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rails/request_middleware.rb

@@ -1,4 +1,4 @@
-# typed: true
+# typed: ignore
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/contrib/sinatra/configuration/settings.rb

@@ -1,7 +1,7 @@
-# typed: false
+# typed: ignore
 
-require 'datadog/appsec/contrib/configuration/settings'
-require 'datadog/appsec/contrib/sinatra/ext'
+require_relative '../../configuration/settings'
+require_relative '../ext'
 
 module Datadog
   module AppSec