ddtrace 1.1.0 → 1.9.0

data/lib/datadog/appsec/contrib/rack/request.rb

@@ -1,4 +1,7 @@
-# typed: true
+# typed: ignore
+
+require_relative '../../../tracing/client_ip'
+require_relative '../../../tracing/contrib/rack/header_collection'
 
 module Datadog
   module AppSec
@@ -47,10 +50,27 @@ module Datadog
           end
 
           def self.form_hash(request)
+            # force form data processing
+            request.POST if request.form_data?
+
             # usually Hash<String,String> but can be a more complex
             # Hash<String,String||Array||Hash> when e.g coming from JSON
             request.env['rack.request.form_hash']
           end
+
+          def self.client_ip(request)
+            remote_ip = request.env['REMOTE_ADDR']
+            headers = Datadog::Tracing::Contrib::Rack::Header::RequestHeaderCollection.new(request.env)
+
+            result = Datadog::Tracing::ClientIp.raw_ip_from_request(headers, remote_ip)
+
+            if result.raw_ip
+              ip = Datadog::Tracing::ClientIp.strip_decorations(result.raw_ip)
+              return unless Datadog::Tracing::ClientIp.valid_ip?(ip)
+
+              ip
+            end
+          end
         end
       end
     end

data/lib/datadog/appsec/contrib/rack/request_body_middleware.rb

@@ -1,7 +1,7 @@
 # typed: ignore
 
-require 'datadog/appsec/instrumentation/gateway'
-require 'datadog/appsec/assets'
+require_relative '../../instrumentation/gateway'
+require_relative '../../response'
 
 module Datadog
   module AppSec
@@ -29,7 +29,7 @@ module Datadog
             end
 
             if request_response && request_response.any? { |action, _event| action == :block }
-              request_return = [403, { 'Content-Type' => 'text/html' }, [Datadog::AppSec::Assets.blocked]]
+              request_return = AppSec::Response.negotiate(env).to_rack
             end
 
             request_return

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -2,9 +2,12 @@
 
 require 'json'
 
-require 'datadog/appsec/instrumentation/gateway'
-require 'datadog/appsec/processor'
-require 'datadog/appsec/assets'
+require_relative '../../instrumentation/gateway'
+require_relative '../../processor'
+require_relative '../../response'
+
+require_relative '../../../tracing/client_ip'
+require_relative '../../../tracing/contrib/rack/header_collection'
 
 module Datadog
   module AppSec
@@ -21,7 +24,7 @@ module Datadog
           end
 
           def call(env)
-            return @app.call(env) unless @processor.ready?
+            return @app.call(env) unless Datadog.configuration.appsec.enabled && @processor.ready?
 
             # TODO: handle exceptions, except for @app.call
 
@@ -30,14 +33,14 @@ module Datadog
             env['datadog.waf.context'] = context
             request = ::Rack::Request.new(env)
 
-            add_appsec_tags
+            add_appsec_tags(active_trace, active_span, env)
 
             request_return, request_response = Instrumentation.gateway.push('rack.request', request) do
               @app.call(env)
             end
 
             if request_response && request_response.any? { |action, _event| action == :block }
-              request_return = [403, { 'Content-Type' => 'text/html' }, [Datadog::AppSec::Assets.blocked]]
+              request_return = AppSec::Response.negotiate(env).to_rack
             end
 
             response = ::Rack::Response.new(request_return[2], request_return[0], request_return[1])
@@ -56,7 +59,8 @@ module Datadog
 
             request_return
           ensure
-            add_waf_runtime_tags(context) if context
+            add_waf_runtime_tags(active_trace, context) if context
+            context.finalize if context
           end
 
           private
@@ -69,41 +73,64 @@ module Datadog
             Datadog::Tracing.active_trace
           end
 
-          def add_appsec_tags
-            return unless active_trace
+          def active_span
+            # TODO: factor out tracing availability detection
+
+            return unless defined?(Datadog::Tracing)
+
+            Datadog::Tracing.active_span
+          end
+
+          def add_appsec_tags(trace, span, env)
+            return unless trace
+
+            trace.set_tag('_dd.appsec.enabled', 1)
+            trace.set_tag('_dd.runtime_family', 'ruby')
+            trace.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
 
-            active_trace.set_tag('_dd.appsec.enabled', 1)
-            active_trace.set_tag('_dd.runtime_family', 'ruby')
-            active_trace.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
+            if span && span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_CLIENT_IP).nil?
+              request_header_collection = Datadog::Tracing::Contrib::Rack::Header::RequestHeaderCollection.new(env)
+
+              # always collect client ip, as this is part of AppSec provided functionality
+              Datadog::Tracing::ClientIp.set_client_ip_tag!(
+                span,
+                headers: request_header_collection,
+                remote_ip: env['REMOTE_ADDR']
+              )
+            end
 
             if @processor.ruleset_info
-              active_trace.set_tag('_dd.appsec.event_rules.version', @processor.ruleset_info[:version])
+              trace.set_tag('_dd.appsec.event_rules.version', @processor.ruleset_info[:version])
 
               unless @oneshot_tags_sent
                 # Small race condition, but it's inoccuous: worst case the tags
                 # are sent a couple of times more than expected
                 @oneshot_tags_sent = true
 
-                active_trace.set_tag('_dd.appsec.event_rules.loaded', @processor.ruleset_info[:loaded].to_f)
-                active_trace.set_tag('_dd.appsec.event_rules.error_count', @processor.ruleset_info[:failed].to_f)
-                active_trace.set_tag('_dd.appsec.event_rules.errors', JSON.dump(@processor.ruleset_info[:errors]))
-                active_trace.set_tag('_dd.appsec.event_rules.addresses', JSON.dump(@processor.addresses))
+                trace.set_tag('_dd.appsec.event_rules.loaded', @processor.ruleset_info[:loaded].to_f)
+                trace.set_tag('_dd.appsec.event_rules.error_count', @processor.ruleset_info[:failed].to_f)
+                trace.set_tag('_dd.appsec.event_rules.errors', JSON.dump(@processor.ruleset_info[:errors]))
+                trace.set_tag('_dd.appsec.event_rules.addresses', JSON.dump(@processor.addresses))
 
                 # Ensure these tags reach the backend
-                active_trace.keep!
+                trace.keep!
+                trace.set_tag(
+                  Datadog::Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
+                  Datadog::Tracing::Sampling::Ext::Decision::ASM
+                )
               end
             end
           end
 
-          def add_waf_runtime_tags(context)
-            return unless active_trace
+          def add_waf_runtime_tags(trace, context)
+            return unless trace
             return unless context
 
-            active_trace.set_tag('_dd.appsec.waf.timeouts', context.timeouts)
+            trace.set_tag('_dd.appsec.waf.timeouts', context.timeouts)
 
             # these tags expect time in us
-            active_trace.set_tag('_dd.appsec.waf.duration', context.time_ns / 1000.0)
-            active_trace.set_tag('_dd.appsec.waf.duration_ext', context.time_ext_ns / 1000.0)
+            trace.set_tag('_dd.appsec.waf.duration', context.time_ns / 1000.0)
+            trace.set_tag('_dd.appsec.waf.duration_ext', context.time_ext_ns / 1000.0)
           end
         end
       end

data/lib/datadog/appsec/contrib/rack/response.rb

@@ -1,4 +1,4 @@
-# typed: true
+# typed: ignore
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/contrib/rails/configuration/settings.rb

@@ -1,7 +1,7 @@
-# typed: false
+# typed: ignore
 
-require 'datadog/appsec/contrib/configuration/settings'
-require 'datadog/appsec/contrib/rails/ext'
+require_relative '../../configuration/settings'
+require_relative '../ext'
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/contrib/rails/ext.rb

@@ -1,4 +1,4 @@
-# typed: true
+# typed: ignore
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/contrib/rails/framework.rb

@@ -1,4 +1,4 @@
-# typed: true
+# typed: ignore
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb

@@ -1,9 +1,9 @@
-# typed: false
+# typed: ignore
 
-require 'datadog/appsec/instrumentation/gateway'
-require 'datadog/appsec/reactive/operation'
-require 'datadog/appsec/contrib/rails/reactive/action'
-require 'datadog/appsec/event'
+require_relative '../../../instrumentation/gateway'
+require_relative '../../../reactive/operation'
+require_relative '../reactive/action'
+require_relative '../../../event'
 
 module Datadog
   module AppSec
@@ -13,7 +13,7 @@ module Datadog
           # Watcher for Rails gateway events
           module Watcher
             def self.watch
-              Instrumentation.gateway.watch('rails.request.action') do |stack, request|
+              Instrumentation.gateway.watch('rails.request.action', :appsec) do |stack, request|
                 block = false
                 event = nil
                 waf_context = request.env['datadog.waf.context']
@@ -22,23 +22,24 @@ module Datadog
                   trace = active_trace
                   span = active_span
 
-                  Rails::Reactive::Action.subscribe(op, waf_context) do |action, result, _block|
-                    record = [:block, :monitor].include?(action)
-                    if record
+                  Rails::Reactive::Action.subscribe(op, waf_context) do |result, _block|
+                    if result.status == :match
                       # TODO: should this hash be an Event instance instead?
                       event = {
                         waf_result: result,
                         trace: trace,
                         span: span,
                         request: request,
-                        action: action
+                        actions: result.actions
                       }
 
+                      span.set_tag('appsec.event', 'true') if span
+
                       waf_context.events << event
                     end
                   end
 
-                  _action, _result, block = Rails::Reactive::Action.publish(op, request)
+                  _result, block = Rails::Reactive::Action.publish(op, request)
                 end
 
                 next [nil, [[:block, event]]] if block

data/lib/datadog/appsec/contrib/rails/integration.rb

@@ -1,10 +1,10 @@
 # typed: ignore
 
-require 'datadog/appsec/contrib/integration'
+require_relative '../integration'
 
-require 'datadog/appsec/contrib/rails/configuration/settings'
-require 'datadog/appsec/contrib/rails/patcher'
-require 'datadog/appsec/contrib/rails/request_middleware'
+require_relative 'configuration/settings'
+require_relative 'patcher'
+require_relative 'request_middleware'
 
 module Datadog
   module AppSec
@@ -19,7 +19,7 @@ module Datadog
           register_as :rails, auto_patch: false
 
           def self.version
-            Gem.loaded_specs['rails'] && Gem.loaded_specs['rails'].version
+            Gem.loaded_specs['railties'] && Gem.loaded_specs['railties'].version
           end
 
           def self.loaded?

data/lib/datadog/appsec/contrib/rails/patcher.rb

@@ -1,14 +1,15 @@
 # typed: ignore
 
-require 'datadog/core/utils/only_once'
+require_relative '../../../core/utils/only_once'
 
-require 'datadog/appsec/contrib/patcher'
-require 'datadog/appsec/contrib/rails/framework'
-require 'datadog/appsec/contrib/rack/request_middleware'
-require 'datadog/appsec/contrib/rack/request_body_middleware'
-require 'datadog/appsec/contrib/rails/gateway/watcher'
+require_relative '../patcher'
+require_relative 'framework'
+require_relative '../../response'
+require_relative '../rack/request_middleware'
+require_relative '../rack/request_body_middleware'
+require_relative 'gateway/watcher'
 
-require 'datadog/tracing/contrib/rack/middlewares'
+require_relative '../../../tracing/contrib/rack/middlewares'
 
 module Datadog
   module AppSec
@@ -58,8 +59,10 @@ module Datadog
           def add_middleware(app)
             # Add trace middleware
             if include_middleware?(Datadog::Tracing::Contrib::Rack::TraceMiddleware, app)
-              app.middleware.insert_after(Datadog::Tracing::Contrib::Rack::TraceMiddleware,
-                                          Datadog::AppSec::Contrib::Rack::RequestMiddleware)
+              app.middleware.insert_after(
+                Datadog::Tracing::Contrib::Rack::TraceMiddleware,
+                Datadog::AppSec::Contrib::Rack::RequestMiddleware
+              )
             else
               app.middleware.insert_before(0, Datadog::AppSec::Contrib::Rack::RequestMiddleware)
             end
@@ -81,9 +84,7 @@ module Datadog
               end
 
               if request_response && request_response.any? { |action, _event| action == :block }
-                @_response = ::ActionDispatch::Response.new(403,
-                                                            { 'Content-Type' => 'text/html' },
-                                                            [Datadog::AppSec::Assets.blocked])
+                @_response = AppSec::Response.negotiate(env).to_action_dispatch_response
                 request_return = @_response.body
               end
 
@@ -92,7 +93,7 @@ module Datadog
           end
 
           def patch_process_action
-            ActionController::Instrumentation.prepend(ProcessActionPatch)
+            ::ActionController::Metal.prepend(ProcessActionPatch)
           end
 
           def include_middleware?(middleware, app)

data/lib/datadog/appsec/contrib/rails/reactive/action.rb

@@ -1,6 +1,6 @@
-# typed: true
+# typed: ignore
 
-require 'datadog/appsec/contrib/rails/request'
+require_relative '../request'
 
 module Datadog
   module AppSec
@@ -36,27 +36,27 @@ module Datadog
                 }
 
                 waf_timeout = Datadog::AppSec.settings.waf_timeout
-                action, result = waf_context.run(waf_args, waf_timeout)
+                result = waf_context.run(waf_args, waf_timeout)
 
                 Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
 
-                # TODO: encapsulate return array in a type
-                case action
-                when :monitor
+                case result.status
+                when :match
                   Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, false]
-                when :block
-                  Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, true]
-                  throw(:block, [action, result, true])
-                when :good
+
+                  block = result.actions.include?('block')
+
+                  yield [result, block]
+
+                  throw(:block, [result, true]) if block
+                when :ok
                   Datadog.logger.debug { "WAF OK: #{result.inspect}" }
                 when :invalid_call
                   Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
                 when :invalid_rule, :invalid_flow, :no_rule
                   Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
                 else
-                  Datadog.logger.debug { "WAF UNKNOWN: #{action.inspect} #{result.inspect}" }
+                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
                 end
               end
             end

data/lib/datadog/appsec/contrib/rails/request.rb

@@ -1,4 +1,4 @@
-# typed: true
+# typed: ignore
 
 module Datadog
   module AppSec
@@ -7,6 +7,9 @@ module Datadog
         # Normalized extration of data from ActionDispatch::Request
         module Request
           def self.parsed_body(request)
+            # force body parameter parsing, which is done lazily by Rails
+            request.parameters
+
             # usually Hash<String,String> but can be a more complex
             # Hash<String,String||Array||Hash> when e.g coming from JSON or
             # with Rails advanced param square bracket parsing

data/lib/datadog/appsec/contrib/rails/request_middleware.rb

@@ -1,4 +1,4 @@
-# typed: true
+# typed: ignore
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/contrib/sinatra/configuration/settings.rb

@@ -1,7 +1,7 @@
-# typed: false
+# typed: ignore
 
-require 'datadog/appsec/contrib/configuration/settings'
-require 'datadog/appsec/contrib/sinatra/ext'
+require_relative '../../configuration/settings'
+require_relative '../ext'
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/contrib/sinatra/ext.rb

@@ -1,4 +1,4 @@
-# typed: true
+# typed: ignore
 
 module Datadog
   module AppSec
@@ -8,6 +8,7 @@ module Datadog
         module Ext
           APP = 'sinatra'.freeze
           ENV_ENABLED = 'DD_TRACE_SINATRA_ENABLED'.freeze
+          ROUTE_INTERRUPT = :datadog_appsec_contrib_sinatra_route_interrupt
         end
       end
     end

data/lib/datadog/appsec/contrib/sinatra/framework.rb

@@ -1,4 +1,4 @@
-# typed: true
+# typed: ignore
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb

@@ -1,21 +1,21 @@
-# typed: false
+# typed: ignore
 
-require 'datadog/appsec/instrumentation/gateway'
-require 'datadog/appsec/reactive/operation'
-require 'datadog/appsec/contrib/rack/reactive/request_body'
-require 'datadog/appsec/contrib/sinatra/reactive/routed'
-require 'datadog/appsec/event'
+require_relative '../../../instrumentation/gateway'
+require_relative '../../../reactive/operation'
+require_relative '../../rack/reactive/request_body'
+require_relative '../reactive/routed'
+require_relative '../../../event'
 
 module Datadog
   module AppSec
     module Contrib
       module Sinatra
         module Gateway
-          # Watcher for Rails gateway events
+          # Watcher for Sinatra gateway events
           module Watcher
             # rubocop:disable Metrics/MethodLength
             def self.watch
-              Instrumentation.gateway.watch('sinatra.request.dispatch') do |stack, request|
+              Instrumentation.gateway.watch('sinatra.request.dispatch', :appsec) do |stack, request|
                 block = false
                 event = nil
                 waf_context = request.env['datadog.waf.context']
@@ -24,23 +24,24 @@ module Datadog
                   trace = active_trace
                   span = active_span
 
-                  Rack::Reactive::RequestBody.subscribe(op, waf_context) do |action, result, _block|
-                    record = [:block, :monitor].include?(action)
-                    if record
+                  Rack::Reactive::RequestBody.subscribe(op, waf_context) do |result, _block|
+                    if result.status == :match
                       # TODO: should this hash be an Event instance instead?
                       event = {
                         waf_result: result,
                         trace: trace,
                         span: span,
                         request: request,
-                        action: action
+                        actions: result.actions
                       }
 
+                      span.set_tag('appsec.event', 'true') if span
+
                       waf_context.events << event
                     end
                   end
 
-                  _action, _result, block = Rack::Reactive::RequestBody.publish(op, request)
+                  _result, block = Rack::Reactive::RequestBody.publish(op, request)
                 end
 
                 next [nil, [[:block, event]]] if block
@@ -55,7 +56,7 @@ module Datadog
                 [ret, res]
               end
 
-              Instrumentation.gateway.watch('sinatra.request.routed') do |stack, (request, route_params)|
+              Instrumentation.gateway.watch('sinatra.request.routed', :appsec) do |stack, (request, route_params)|
                 block = false
                 event = nil
                 waf_context = request.env['datadog.waf.context']
@@ -64,23 +65,24 @@ module Datadog
                   trace = active_trace
                   span = active_span
 
-                  Sinatra::Reactive::Routed.subscribe(op, waf_context) do |action, result, _block|
-                    record = [:block, :monitor].include?(action)
-                    if record
+                  Sinatra::Reactive::Routed.subscribe(op, waf_context) do |result, _block|
+                    if result.status == :match
                       # TODO: should this hash be an Event instance instead?
                       event = {
                         waf_result: result,
                         trace: trace,
                         span: span,
                         request: request,
-                        action: action
+                        actions: result.actions
                       }
 
+                      span.set_tag('appsec.event', 'true') if span
+
                       waf_context.events << event
                     end
                   end
 
-                  _action, _result, block = Sinatra::Reactive::Routed.publish(op, [request, route_params])
+                  _result, block = Sinatra::Reactive::Routed.publish(op, [request, route_params])
                 end
 
                 next [nil, [[:block, event]]] if block

data/lib/datadog/appsec/contrib/sinatra/integration.rb

@@ -1,10 +1,10 @@
 # typed: ignore
 
-require 'datadog/appsec/contrib/integration'
+require_relative '../integration'
 
-require 'datadog/appsec/contrib/sinatra/configuration/settings'
-require 'datadog/appsec/contrib/sinatra/patcher'
-require 'datadog/appsec/contrib/sinatra/request_middleware'
+require_relative 'configuration/settings'
+require_relative 'patcher'
+require_relative 'request_middleware'
 
 module Datadog
   module AppSec

data/lib/datadog/appsec/contrib/sinatra/patcher.rb

@@ -1,12 +1,13 @@
 # typed: ignore
 
-require 'datadog/tracing/contrib/rack/middlewares'
+require_relative '../../../tracing/contrib/rack/middlewares'
 
-require 'datadog/appsec/contrib/patcher'
-require 'datadog/appsec/contrib/rack/request_middleware'
-require 'datadog/appsec/contrib/sinatra/framework'
-require 'datadog/appsec/contrib/sinatra/gateway/watcher'
-require 'datadog/tracing/contrib/sinatra/framework'
+require_relative '../patcher'
+require_relative '../../response'
+require_relative '../rack/request_middleware'
+require_relative 'framework'
+require_relative 'gateway/watcher'
+require_relative '../../../tracing/contrib/sinatra/framework'
 
 module Datadog
   module AppSec
@@ -31,9 +32,11 @@ module Datadog
               tracing_middleware = Datadog::Tracing::Contrib::Rack::TraceMiddleware
 
               if tracing_sinatra_framework.include_middleware?(tracing_middleware, builder)
-                tracing_sinatra_framework.add_middleware_after(tracing_middleware,
-                                                               Datadog::AppSec::Contrib::Rack::RequestMiddleware,
-                                                               builder)
+                tracing_sinatra_framework.add_middleware_after(
+                  tracing_middleware,
+                  Datadog::AppSec::Contrib::Rack::RequestMiddleware,
+                  builder
+                )
               else
                 tracing_sinatra_framework.add_middleware(Datadog::AppSec::Contrib::Rack::RequestMiddleware, builder)
               end
@@ -55,13 +58,12 @@ module Datadog
             # TODO: handle exceptions, except for super
 
             request_return, request_response = Instrumentation.gateway.push('sinatra.request.dispatch', request) do
-              super
+              # handle process_route interruption
+              catch(Ext::ROUTE_INTERRUPT) { super }
             end
 
             if request_response && request_response.any? { |action, _event| action == :block }
-              self.response = ::Sinatra::Response.new([Datadog::AppSec::Assets.blocked],
-                                                      403,
-                                                      { 'Content-Type' => 'text/html' })
+              self.response = AppSec::Response.negotiate(env).to_sinatra_response
               request_return = nil
             end
 
@@ -90,9 +92,14 @@ module Datadog
               # At this point params has both route params and normal params.
               route_params = params.each.with_object({}) { |(k, v), h| h[k] = v unless base_params.key?(k) }
 
-              Instrumentation.gateway.push('sinatra.request.routed', [request, route_params])
+              _, request_response = Instrumentation.gateway.push('sinatra.request.routed', [request, route_params])
 
-              # TODO: handle block
+              if request_response && request_response.any? { |action, _event| action == :block }
+                self.response = AppSec::Response.negotiate(env).to_sinatra_response
+
+                # interrupt request and return response to dispatch! for consistency
+                throw(Ext::ROUTE_INTERRUPT, response)
+              end
 
               yield(*args)
             end