ddtrace 1.4.2 → 1.23.3

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -1,15 +1,30 @@
-# typed: ignore
-
 require 'json'
 
+require_relative 'gateway/request'
+require_relative 'gateway/response'
 require_relative '../../instrumentation/gateway'
 require_relative '../../processor'
-require_relative '../../assets'
+require_relative '../../response'
+
+require_relative '../../../tracing/client_ip'
+require_relative '../../../tracing/contrib/rack/header_collection'
 
 module Datadog
   module AppSec
     module Contrib
       module Rack
+        # Create an array of lowercased headers
+        WAF_VENDOR_HEADERS_TAGS = %w[
+          X-Amzn-Trace-Id
+          Cloudfront-Viewer-Ja3-Fingerprint
+          Cf-Ray
+          X-Cloud-Trace-Context
+          X-Appgw-Trace-id
+          X-SigSci-RequestID
+          X-SigSci-Tags
+          Akamai-User-Risk
+        ].map(&:downcase).freeze
+
         # Topmost Rack middleware for AppSec
         # This should be inserted just below Datadog::Tracing::Contrib::Rack::TraceMiddleware
         class RequestMiddleware
@@ -17,50 +32,101 @@ module Datadog
             @app = app
 
             @oneshot_tags_sent = false
-            @processor = Datadog::AppSec::Processor.new
+            @rack_headers = {}
           end
 
+          # rubocop:disable Metrics/AbcSize,Metrics/PerceivedComplexity,Metrics/CyclomaticComplexity,Metrics/MethodLength
           def call(env)
-            return @app.call(env) unless @processor.ready?
+            return @app.call(env) unless Datadog::AppSec.enabled?
 
-            # TODO: handle exceptions, except for @app.call
+            boot = Datadog::Core::Remote::Tie.boot
+            Datadog::Core::Remote::Tie::Tracing.tag(boot, active_span)
 
-            context = @processor.new_context
+            processor = nil
+            ready = false
+            scope = nil
 
-            env['datadog.waf.context'] = context
-            request = ::Rack::Request.new(env)
+            # For a given request, keep using the first Rack stack scope for
+            # nested apps. Don't set `context` local variable so that on popping
+            # out of this nested stack we don't finalize the parent's context
+            return @app.call(env) if active_scope(env)
 
-            add_appsec_tags
+            Datadog::AppSec.reconfigure_lock do
+              processor = Datadog::AppSec.processor
 
-            request_return, request_response = Instrumentation.gateway.push('rack.request', request) do
-              @app.call(env)
+              if !processor.nil? && processor.ready?
+                scope = Datadog::AppSec::Scope.activate_scope(active_trace, active_span, processor)
+                env[Datadog::AppSec::Ext::SCOPE_KEY] = scope
+                ready = true
+              end
             end
 
-            if request_response && request_response.any? { |action, _event| action == :block }
-              request_return = [403, { 'Content-Type' => 'text/html' }, [Datadog::AppSec::Assets.blocked]]
+            # TODO: handle exceptions, except for @app.call
+
+            return @app.call(env) unless ready
+
+            gateway_request = Gateway::Request.new(env)
+
+            add_appsec_tags(processor, scope)
+            add_request_tags(scope, env)
+
+            request_return, request_response = catch(::Datadog::AppSec::Ext::INTERRUPT) do
+              Instrumentation.gateway.push('rack.request', gateway_request) do
+                @app.call(env)
+              end
             end
 
-            response = ::Rack::Response.new(request_return[2], request_return[0], request_return[1])
-            response.instance_eval do
-              @waf_context = context
+            if request_response
+              blocked_event = request_response.find { |action, _options| action == :block }
+              request_return = AppSec::Response.negotiate(env, blocked_event.last[:actions]).to_rack if blocked_event
             end
 
-            _response_return, _response_response = Instrumentation.gateway.push('rack.response', response)
+            gateway_response = Gateway::Response.new(
+              request_return[2],
+              request_return[0],
+              request_return[1],
+              scope: scope,
+            )
+
+            _response_return, response_response = Instrumentation.gateway.push('rack.response', gateway_response)
+
+            result = scope.processor_context.extract_schema
+
+            if result
+              scope.processor_context.events << {
+                trace: scope.trace,
+                span: scope.service_entry_span,
+                waf_result: result,
+              }
+            end
 
-            context.events.each do |e|
-              e[:response] ||= response
-              e[:request]  ||= request
+            scope.processor_context.events.each do |e|
+              e[:response] ||= gateway_response
+              e[:request]  ||= gateway_request
             end
 
-            AppSec::Event.record(*context.events)
+            AppSec::Event.record(scope.service_entry_span, *scope.processor_context.events)
+
+            if response_response
+              blocked_event = response_response.find { |action, _options| action == :block }
+              request_return = AppSec::Response.negotiate(env, blocked_event.last[:actions]).to_rack if blocked_event
+            end
 
             request_return
           ensure
-            add_waf_runtime_tags(context) if context
+            if scope
+              add_waf_runtime_tags(scope)
+              Datadog::AppSec::Scope.deactivate_scope
+            end
           end
+          # rubocop:enable Metrics/AbcSize,Metrics/PerceivedComplexity,Metrics/CyclomaticComplexity,Metrics/MethodLength
 
           private
 
+          def active_scope(env)
+            env[Datadog::AppSec::Ext::SCOPE_KEY]
+          end
+
           def active_trace
             # TODO: factor out tracing availability detection
 
@@ -69,41 +135,87 @@ module Datadog
             Datadog::Tracing.active_trace
           end
 
-          def add_appsec_tags
-            return unless active_trace
+          def active_span
+            # TODO: factor out tracing availability detection
+
+            return unless defined?(Datadog::Tracing)
+
+            Datadog::Tracing.active_span
+          end
 
-            active_trace.set_tag('_dd.appsec.enabled', 1)
-            active_trace.set_tag('_dd.runtime_family', 'ruby')
-            active_trace.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
+          def add_appsec_tags(processor, scope)
+            span = scope.service_entry_span
+            trace = scope.trace
 
-            if @processor.ruleset_info
-              active_trace.set_tag('_dd.appsec.event_rules.version', @processor.ruleset_info[:version])
+            return unless trace && span
+
+            span.set_tag('_dd.appsec.enabled', 1)
+            span.set_tag('_dd.runtime_family', 'ruby')
+            span.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
+
+            if processor.diagnostics
+              diagnostics = processor.diagnostics
+
+              span.set_tag('_dd.appsec.event_rules.version', diagnostics['ruleset_version'])
 
               unless @oneshot_tags_sent
                 # Small race condition, but it's inoccuous: worst case the tags
                 # are sent a couple of times more than expected
                 @oneshot_tags_sent = true
 
-                active_trace.set_tag('_dd.appsec.event_rules.loaded', @processor.ruleset_info[:loaded].to_f)
-                active_trace.set_tag('_dd.appsec.event_rules.error_count', @processor.ruleset_info[:failed].to_f)
-                active_trace.set_tag('_dd.appsec.event_rules.errors', JSON.dump(@processor.ruleset_info[:errors]))
-                active_trace.set_tag('_dd.appsec.event_rules.addresses', JSON.dump(@processor.addresses))
+                span.set_tag('_dd.appsec.event_rules.loaded', diagnostics['rules']['loaded'].size.to_f)
+                span.set_tag('_dd.appsec.event_rules.error_count', diagnostics['rules']['failed'].size.to_f)
+                span.set_tag('_dd.appsec.event_rules.errors', JSON.dump(diagnostics['rules']['errors']))
+                span.set_tag('_dd.appsec.event_rules.addresses', JSON.dump(processor.addresses))
 
                 # Ensure these tags reach the backend
-                active_trace.keep!
+                trace.keep!
+                trace.set_tag(
+                  Datadog::Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
+                  Datadog::Tracing::Sampling::Ext::Decision::ASM
+                )
               end
             end
           end
 
-          def add_waf_runtime_tags(context)
-            return unless active_trace
-            return unless context
+          def add_request_tags(scope, env)
+            span = scope.service_entry_span
 
-            active_trace.set_tag('_dd.appsec.waf.timeouts', context.timeouts)
+            return unless span
+
+            # Always add WAF vendors headers
+            WAF_VENDOR_HEADERS_TAGS.each do |lowercase_header|
+              rack_header = to_rack_header(lowercase_header)
+              span.set_tag("http.request.headers.#{lowercase_header}", env[rack_header]) if env[rack_header]
+            end
+
+            if span && span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_CLIENT_IP).nil?
+              request_header_collection = Datadog::Tracing::Contrib::Rack::Header::RequestHeaderCollection.new(env)
+
+              # always collect client ip, as this is part of AppSec provided functionality
+              Datadog::Tracing::ClientIp.set_client_ip_tag!(
+                span,
+                headers: request_header_collection,
+                remote_ip: env['REMOTE_ADDR']
+              )
+            end
+          end
+
+          def add_waf_runtime_tags(scope)
+            span = scope.service_entry_span
+            context = scope.processor_context
+
+            return unless span && context
+
+            span.set_tag('_dd.appsec.waf.timeouts', context.timeouts)
 
             # these tags expect time in us
-            active_trace.set_tag('_dd.appsec.waf.duration', context.time_ns / 1000.0)
-            active_trace.set_tag('_dd.appsec.waf.duration_ext', context.time_ext_ns / 1000.0)
+            span.set_tag('_dd.appsec.waf.duration', context.time_ns / 1000.0)
+            span.set_tag('_dd.appsec.waf.duration_ext', context.time_ext_ns / 1000.0)
+          end
+
+          def to_rack_header(header)
+            @rack_headers[header] ||= Datadog::Tracing::Contrib::Rack::Header.to_rack_header(header)
           end
         end
       end

data/lib/datadog/appsec/contrib/rails/ext.rb

@@ -1,13 +1,11 @@
-# typed: true
+# frozen_string_literal: true
 
 module Datadog
   module AppSec
     module Contrib
       module Rails
-        # Rack integration constants
+        # Rails integration constants
         module Ext
-          APP = 'rails'.freeze
-          ENV_ENABLED = 'DD_TRACE_RAILS_ENABLED'.freeze
         end
       end
     end

data/lib/datadog/appsec/contrib/rails/framework.rb

@@ -1,4 +1,4 @@
-# typed: true
+# frozen_string_literal: true
 
 module Datadog
   module AppSec
@@ -7,21 +7,7 @@ module Datadog
         # Rails specific framework tie
         module Framework
           def self.setup
-            Datadog::AppSec.configure do |datadog_config|
-              rails_config = config_with_defaults(datadog_config)
-              activate_rack!(datadog_config, rails_config)
-            end
-          end
-
-          def self.config_with_defaults(datadog_config)
-            datadog_config[:rails]
-          end
-
-          # Apply relevant configuration from Sinatra to Rack
-          def self.activate_rack!(datadog_config, sinatra_config)
-            datadog_config.instrument(
-              :rack,
-            )
+            Datadog.configuration.appsec.instrument(:rack)
           end
         end
       end

data/lib/datadog/appsec/contrib/rails/gateway/request.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require_relative '../../../instrumentation/gateway/argument'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Rails
+        module Gateway
+          # Gateway Request argument. Normalized extration of data from ActionDispatch::Request
+          class Request < Instrumentation::Gateway::Argument
+            attr_reader :request
+
+            def initialize(request)
+              super()
+              @request = request
+            end
+
+            def env
+              request.env
+            end
+
+            def headers
+              request.headers
+            end
+
+            def host
+              request.host
+            end
+
+            def user_agent
+              request.user_agent
+            end
+
+            def remote_addr
+              request.remote_addr
+            end
+
+            def parsed_body
+              # force body parameter parsing, which is done lazily by Rails
+              request.parameters
+
+              # usually Hash<String,String> but can be a more complex
+              # Hash<String,String||Array||Hash> when e.g coming from JSON or
+              # with Rails advanced param square bracket parsing
+              body = request.env['action_dispatch.request.request_parameters']
+
+              return if body.nil?
+
+              body.reject do |k, _v|
+                request.env['action_dispatch.request.path_parameters'].key?(k)
+              end
+            end
+
+            def route_params
+              excluded = [:controller, :action]
+
+              request.env['action_dispatch.request.path_parameters'].reject do |k, _v|
+                excluded.include?(k)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb

@@ -1,5 +1,3 @@
-# typed: false
-
 require_relative '../../../instrumentation/gateway'
 require_relative '../../../reactive/operation'
 require_relative '../reactive/action'
@@ -12,65 +10,55 @@ module Datadog
         module Gateway
           # Watcher for Rails gateway events
           module Watcher
-            def self.watch
-              Instrumentation.gateway.watch('rails.request.action') do |stack, request|
-                block = false
-                event = nil
-                waf_context = request.env['datadog.waf.context']
-
-                AppSec::Reactive::Operation.new('rails.request.action') do |op|
-                  trace = active_trace
-                  span = active_span
+            class << self
+              def watch
+                gateway = Instrumentation.gateway
 
-                  Rails::Reactive::Action.subscribe(op, waf_context) do |action, result, _block|
-                    record = [:block, :monitor].include?(action)
-                    if record
-                      # TODO: should this hash be an Event instance instead?
-                      event = {
-                        waf_result: result,
-                        trace: trace,
-                        span: span,
-                        request: request,
-                        action: action
-                      }
+                watch_request_action(gateway)
+              end
 
-                      waf_context.events << event
+              def watch_request_action(gateway = Instrumentation.gateway)
+                gateway.watch('rails.request.action', :appsec) do |stack, gateway_request|
+                  block = false
+
+                  event = nil
+                  scope = gateway_request.env[Datadog::AppSec::Ext::SCOPE_KEY]
+
+                  AppSec::Reactive::Operation.new('rails.request.action') do |op|
+                    Rails::Reactive::Action.subscribe(op, scope.processor_context) do |result|
+                      if result.status == :match
+                        # TODO: should this hash be an Event instance instead?
+                        event = {
+                          waf_result: result,
+                          trace: scope.trace,
+                          span: scope.service_entry_span,
+                          request: gateway_request,
+                          actions: result.actions
+                        }
+
+                        if scope.service_entry_span
+                          scope.service_entry_span.set_tag('appsec.blocked', 'true') if result.actions.include?('block')
+                          scope.service_entry_span.set_tag('appsec.event', 'true')
+                        end
+
+                        scope.processor_context.events << event
+                      end
                     end
+
+                    block = Rails::Reactive::Action.publish(op, gateway_request)
                   end
 
-                  _action, _result, block = Rails::Reactive::Action.publish(op, request)
-                end
+                  next [nil, [[:block, event]]] if block
 
-                next [nil, [[:block, event]]] if block
+                  ret, res = stack.call(gateway_request.request)
 
-                ret, res = stack.call(request)
+                  if event
+                    res ||= []
+                    res << [:monitor, event]
+                  end
 
-                if event
-                  res ||= []
-                  res << [:monitor, event]
+                  [ret, res]
                 end
-
-                [ret, res]
-              end
-            end
-
-            class << self
-              private
-
-              def active_trace
-                # TODO: factor out tracing availability detection
-
-                return unless defined?(Datadog::Tracing)
-
-                Datadog::Tracing.active_trace
-              end
-
-              def active_span
-                # TODO: factor out tracing availability detection
-
-                return unless defined?(Datadog::Tracing)
-
-                Datadog::Tracing.active_span
               end
             end
           end

data/lib/datadog/appsec/contrib/rails/integration.rb

@@ -1,8 +1,5 @@
-# typed: ignore
-
 require_relative '../integration'
 
-require_relative 'configuration/settings'
 require_relative 'patcher'
 require_relative 'request_middleware'
 
@@ -19,7 +16,7 @@ module Datadog
           register_as :rails, auto_patch: false
 
           def self.version
-            Gem.loaded_specs['rails'] && Gem.loaded_specs['rails'].version
+            Gem.loaded_specs['railties'] && Gem.loaded_specs['railties'].version
           end
 
           def self.loaded?
@@ -34,10 +31,6 @@ module Datadog
             true
           end
 
-          def default_configuration
-            Configuration::Settings.new
-          end
-
           def patcher
             Patcher
           end

data/lib/datadog/appsec/contrib/rails/patcher.rb

@@ -1,12 +1,12 @@
-# typed: ignore
-
 require_relative '../../../core/utils/only_once'
 
 require_relative '../patcher'
 require_relative 'framework'
+require_relative '../../response'
 require_relative '../rack/request_middleware'
 require_relative '../rack/request_body_middleware'
 require_relative 'gateway/watcher'
+require_relative 'gateway/request'
 
 require_relative '../../../tracing/contrib/rack/middlewares'
 
@@ -33,19 +33,19 @@ module Datadog
 
           def patch
             Gateway::Watcher.watch
-            patch_before_intialize
-            patch_after_intialize
+            patch_before_initialize
+            patch_after_initialize
 
             Patcher.instance_variable_set(:@patched, true)
           end
 
-          def patch_before_intialize
+          def patch_before_initialize
             ::ActiveSupport.on_load(:before_initialize) do
-              Datadog::AppSec::Contrib::Rails::Patcher.before_intialize(self)
+              Datadog::AppSec::Contrib::Rails::Patcher.before_initialize(self)
             end
           end
 
-          def before_intialize(app)
+          def before_initialize(app)
             BEFORE_INITIALIZE_ONLY_ONCE_PER_APP[app].run do
               # Middleware must be added before the application is initialized.
               # Otherwise the middleware stack will be frozen.
@@ -72,23 +72,26 @@ module Datadog
             def process_action(*args)
               env = request.env
 
-              context = env['datadog.waf.context']
+              context = env[Datadog::AppSec::Ext::SCOPE_KEY]
 
               return super unless context
 
               # TODO: handle exceptions, except for super
 
-              request_return, request_response = Instrumentation.gateway.push('rails.request.action', request) do
+              gateway_request = Gateway::Request.new(request)
+              request_return, request_response = Instrumentation.gateway.push('rails.request.action', gateway_request) do
                 super
               end
 
-              if request_response && request_response.any? { |action, _event| action == :block }
-                @_response = ::ActionDispatch::Response.new(
-                  403,
-                  { 'Content-Type' => 'text/html' },
-                  [Datadog::AppSec::Assets.blocked]
-                )
-                request_return = @_response.body
+              if request_response
+                blocked_event = request_response.find { |action, _options| action == :block }
+                if blocked_event
+                  @_response = AppSec::Response.negotiate(
+                    env,
+                    blocked_event.last[:actions]
+                  ).to_action_dispatch_response
+                  request_return = @_response.body
+                end
               end
 
               request_return
@@ -96,7 +99,7 @@ module Datadog
           end
 
           def patch_process_action
-            ActionController::Instrumentation.prepend(ProcessActionPatch)
+            ::ActionController::Metal.prepend(ProcessActionPatch)
           end
 
           def include_middleware?(middleware, app)
@@ -137,13 +140,13 @@ module Datadog
             Datadog.logger.debug { 'Rails middlewares: ' << app.middleware.map(&:inspect).inspect }
           end
 
-          def patch_after_intialize
+          def patch_after_initialize
             ::ActiveSupport.on_load(:after_initialize) do
-              Datadog::AppSec::Contrib::Rails::Patcher.after_intialize(self)
+              Datadog::AppSec::Contrib::Rails::Patcher.after_initialize(self)
             end
           end
 
-          def after_intialize(app)
+          def after_initialize(app)
             AFTER_INITIALIZE_ONLY_ONCE_PER_APP[app].run do
               # Finish configuring the tracer after the application is initialized.
               # We need to wait for some things, like application name, middleware stack, etc.