ddtrace 1.14.0 → 1.16.0

data/lib/datadog/appsec/configuration/settings.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../../core/utils/duration'
+require_relative '../sample_rate'
 
 module Datadog
   module AppSec
@@ -25,7 +26,7 @@ module Datadog
           add_settings!(base)
         end
 
-        # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength
+        # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
         def self.add_settings!(base)
           base.class_eval do
             settings :appsec do
@@ -95,6 +96,46 @@ module Datadog
                 o.default DEFAULT_OBFUSCATOR_VALUE_REGEX
               end
 
+              settings :block do
+                settings :templates do
+                  option :html do |o|
+                    o.env 'DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML'
+                    o.type :string, nilable: true
+                    o.setter do |value|
+                      if value
+                        raise(ArgumentError, "appsec.templates.html: file not found: #{value}") unless File.exist?(value)
+
+                        File.open(value, 'rb', &:read) || ''
+                      end
+                    end
+                  end
+
+                  option :json do |o|
+                    o.env 'DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON'
+                    o.type :string, nilable: true
+                    o.setter do |value|
+                      if value
+                        raise(ArgumentError, "appsec.templates.json: file not found: #{value}") unless File.exist?(value)
+
+                        File.open(value, 'rb', &:read) || ''
+                      end
+                    end
+                  end
+
+                  option :text do |o|
+                    o.env 'DD_APPSEC_HTTP_BLOCKED_TEMPLATE_TEXT'
+                    o.type :string, nilable: true
+                    o.setter do |value|
+                      if value
+                        raise(ArgumentError, "appsec.templates.text: file not found: #{value}") unless File.exist?(value)
+
+                        File.open(value, 'rb', &:read) || ''
+                      end
+                    end
+                  end
+                end
+              end
+
               settings :track_user_events do
                 option :enabled do |o|
                   o.default true
@@ -129,10 +170,34 @@ module Datadog
                   end
                 end
               end
+
+              settings :api_security do
+                option :enabled do |o|
+                  o.type :bool
+                  o.env 'DD_EXPERIMENTAL_API_SECURITY_ENABLED'
+                  o.default false
+                end
+
+                option :sample_rate do |o|
+                  o.type :float
+                  o.env 'DD_API_SECURITY_REQUEST_SAMPLE_RATE'
+                  o.default 0.1
+                  o.setter do |value|
+                    value = 1 if value > 1
+                    SampleRate.new(value)
+                  end
+                end
+              end
+
+              option :parse_response_body do |o|
+                o.type :bool
+                o.env 'DD_API_SECURITY_PARSE_RESPONSE_BODY'
+                o.default true
+              end
             end
           end
         end
-        # rubocop:enable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength
+        # rubocop:enable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
       end
     end
   end

data/lib/datadog/appsec/contrib/rack/gateway/request.rb

@@ -40,9 +40,13 @@ module Datadog
             end
 
             def headers
-              request.env.each_with_object({}) do |(k, v), h|
-                h[k.gsub(/^HTTP_/, '').downcase.tr('_', '-')] = v if k =~ /^HTTP_/
+              result = request.env.each_with_object({}) do |(k, v), h|
+                h[k.gsub(/^HTTP_/, '').downcase!.tr('_', '-')] = v if k =~ /^HTTP_/
               end
+
+              result['content-type'] = request.content_type if request.content_type
+              result['content-length'] = request.content_length if request.content_length
+              result
             end
 
             def body

data/lib/datadog/appsec/contrib/rack/gateway/response.rb

@@ -19,9 +19,55 @@ module Datadog
               @scope = scope
             end
 
+            def parsed_body
+              return unless Datadog.configuration.appsec.parse_response_body
+
+              unless body.instance_of?(Array)
+                Datadog.logger.debug do
+                  "Response body type unsupported: #{body.class}"
+                end
+                return
+              end
+
+              return unless json_content_type?
+
+              result = ''.dup
+              all_body_parts_are_string = true
+
+              body.each do |body_part|
+                if body_part.is_a?(String)
+                  result.concat(body_part)
+                else
+                  all_body_parts_are_string = false
+                  break
+                end
+              end
+
+              return unless all_body_parts_are_string
+
+              begin
+                JSON.parse(result)
+              rescue JSON::ParserError => e
+                Datadog.logger.debug { "Failed to parse response body. Error #{e.class}. Message #{e.message}" }
+                nil
+              end
+            end
+
             def response
               @response ||= ::Rack::Response.new(body, status, headers)
             end
+
+            private
+
+            VALID_JSON_TYPES = [
+              'application/json',
+              'text/json'
+            ].freeze
+
+            def json_content_type?
+              content_type = headers['content-type']
+              VALID_JSON_TYPES.include?(content_type)
+            end
           end
         end
       end

data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb

@@ -28,7 +28,7 @@ module Datadog
                   scope = gateway_request.env[Datadog::AppSec::Ext::SCOPE_KEY]
 
                   AppSec::Reactive::Operation.new('rack.request') do |op|
-                    Rack::Reactive::Request.subscribe(op, scope.processor_context) do |result, _block|
+                    Rack::Reactive::Request.subscribe(op, scope.processor_context) do |result|
                       if result.status == :match
                         # TODO: should this hash be an Event instance instead?
                         event = {
@@ -48,7 +48,7 @@ module Datadog
                       end
                     end
 
-                    _result, block = Rack::Reactive::Request.publish(op, gateway_request)
+                    block = Rack::Reactive::Request.publish(op, gateway_request)
                   end
 
                   next [nil, [[:block, event]]] if block
@@ -67,11 +67,12 @@ module Datadog
               def watch_response(gateway = Instrumentation.gateway)
                 gateway.watch('rack.response', :appsec) do |stack, gateway_response|
                   block = false
+
                   event = nil
                   scope = gateway_response.scope
 
                   AppSec::Reactive::Operation.new('rack.response') do |op|
-                    Rack::Reactive::Response.subscribe(op, scope.processor_context) do |result, _block|
+                    Rack::Reactive::Response.subscribe(op, scope.processor_context) do |result|
                       if result.status == :match
                         # TODO: should this hash be an Event instance instead?
                         event = {
@@ -91,7 +92,7 @@ module Datadog
                       end
                     end
 
-                    _result, block = Rack::Reactive::Response.publish(op, gateway_response)
+                    block = Rack::Reactive::Response.publish(op, gateway_response)
                   end
 
                   next [nil, [[:block, event]]] if block
@@ -110,11 +111,12 @@ module Datadog
               def watch_request_body(gateway = Instrumentation.gateway)
                 gateway.watch('rack.request.body', :appsec) do |stack, gateway_request|
                   block = false
+
                   event = nil
                   scope = gateway_request.env[Datadog::AppSec::Ext::SCOPE_KEY]
 
                   AppSec::Reactive::Operation.new('rack.request.body') do |op|
-                    Rack::Reactive::RequestBody.subscribe(op, scope.processor_context) do |result, _block|
+                    Rack::Reactive::RequestBody.subscribe(op, scope.processor_context) do |result|
                       if result.status == :match
                         # TODO: should this hash be an Event instance instead?
                         event = {
@@ -134,7 +136,7 @@ module Datadog
                       end
                     end
 
-                    _result, block = Rack::Reactive::RequestBody.publish(op, gateway_request)
+                    block = Rack::Reactive::RequestBody.publish(op, gateway_request)
                   end
 
                   next [nil, [[:block, event]]] if block

data/lib/datadog/appsec/contrib/rack/reactive/request.rb

@@ -30,7 +30,6 @@ module Datadog
               end
             end
 
-            # rubocop:disable Metrics/MethodLength
             def self.subscribe(op, waf_context)
               op.subscribe(*ADDRESSES) do |*values|
                 Datadog.logger.debug { "reacted to #{ADDRESSES.inspect}: #{values.inspect}" }
@@ -61,11 +60,8 @@ module Datadog
                 when :match
                   Datadog.logger.debug { "WAF: #{result.inspect}" }
 
-                  block = result.actions.include?('block')
-
-                  yield [result, block]
-
-                  throw(:block, [result, true]) if block
+                  yield result
+                  throw(:block, true) unless result.actions.empty?
                 when :ok
                   Datadog.logger.debug { "WAF OK: #{result.inspect}" }
                 when :invalid_call
@@ -76,7 +72,6 @@ module Datadog
                   Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
                 end
               end
-              # rubocop:enable Metrics/MethodLength
             end
           end
         end

data/lib/datadog/appsec/contrib/rack/reactive/request_body.rb

@@ -39,11 +39,8 @@ module Datadog
                 when :match
                   Datadog.logger.debug { "WAF: #{result.inspect}" }
 
-                  block = result.actions.include?('block')
-
-                  yield [result, block]
-
-                  throw(:block, [result, true]) if block
+                  yield result
+                  throw(:block, true) unless result.actions.empty?
                 when :ok
                   Datadog.logger.debug { "WAF OK: #{result.inspect}" }
                 when :invalid_call

data/lib/datadog/appsec/contrib/rack/reactive/response.rb

@@ -10,6 +10,7 @@ module Datadog
             ADDRESSES = [
               'response.status',
               'response.headers',
+              'response.body',
             ].freeze
             private_constant :ADDRESSES
 
@@ -17,6 +18,7 @@ module Datadog
               catch(:block) do
                 op.publish('response.status', gateway_response.status)
                 op.publish('response.headers', gateway_response.headers)
+                op.publish('response.body', gateway_response.parsed_body)
 
                 nil
               end
@@ -29,6 +31,7 @@ module Datadog
                 response_status = values[0]
                 response_headers = values[1]
                 response_headers_no_cookies = response_headers.dup.tap { |h| h.delete('set-cookie') }
+                response_body = values[2]
 
                 waf_args = {
                   'server.response.status' => response_status.to_s,
@@ -36,6 +39,8 @@ module Datadog
                   'server.response.headers.no_cookies' => response_headers_no_cookies,
                 }
 
+                waf_args['server.response.body'] = response_body if response_body
+
                 waf_timeout = Datadog.configuration.appsec.waf_timeout
                 result = waf_context.run(waf_args, waf_timeout)
 
@@ -45,11 +50,8 @@ module Datadog
                 when :match
                   Datadog.logger.debug { "WAF: #{result.inspect}" }
 
-                  block = result.actions.include?('block')
-
-                  yield [result, block]
-
-                  throw(:block, [result, true]) if block
+                  yield result
+                  throw(:block, true) unless result.actions.empty?
                 when :ok
                   Datadog.logger.debug { "WAF OK: #{result.inspect}" }
                 when :invalid_call

data/lib/datadog/appsec/contrib/rack/request_body_middleware.rb

@@ -30,8 +30,9 @@ module Datadog
               @app.call(env)
             end
 
-            if request_response && request_response.any? { |action, _event| action == :block }
-              request_return = AppSec::Response.negotiate(env).to_rack
+            if request_response
+              blocked_event = request_response.find { |action, _event| action == :block }
+              request_return = AppSec::Response.negotiate(env, blocked_event.last[:actions]).to_rack if blocked_event
             end
 
             request_return

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -61,19 +61,40 @@ module Datadog
               end
             end
 
-            if request_response && request_response.any? { |action, _event| action == :block }
-              request_return = AppSec::Response.negotiate(env).to_rack
+            if request_response
+              blocked_event = request_response.find { |action, _options| action == :block }
+              request_return = AppSec::Response.negotiate(env, blocked_event.last[:actions]).to_rack if blocked_event
+            end
+
+            if request_return[2].respond_to?(:to_ary)
+              # Following the Rack specification. The response body should only call :each once.
+              # Calling :to_ary returns an array with identical content as the produced when calling :each
+              # replacing request_return[2] with that new value allow us to safely operate on the response body.
+              # On Gateway::Response#parsed_body we might iterate over the reposne body using :each
+              # https://github.com/rack/rack/blob/main/SPEC.rdoc#enumerable-body-
+              consumed_body = request_return[2].to_ary
+              request_return[2] = consumed_body if consumed_body
             end
 
             gateway_response = Gateway::Response.new(
               request_return[2],
               request_return[0],
               request_return[1],
-              scope: scope
+              scope: scope,
             )
 
             _response_return, response_response = Instrumentation.gateway.push('rack.response', gateway_response)
 
+            result = scope.processor_context.extract_schema
+
+            if result
+              scope.processor_context.events << {
+                trace: scope.trace,
+                span: scope.service_entry_span,
+                waf_result: result,
+              }
+            end
+
             scope.processor_context.events.each do |e|
               e[:response] ||= gateway_response
               e[:request]  ||= gateway_request
@@ -81,8 +102,9 @@ module Datadog
 
             AppSec::Event.record(scope.service_entry_span, *scope.processor_context.events)
 
-            if response_response && response_response.any? { |action, _event| action == :block }
-              request_return = AppSec::Response.negotiate(env).to_rack
+            if response_response
+              blocked_event = response_response.find { |action, _options| action == :block }
+              request_return = AppSec::Response.negotiate(env, blocked_event.last[:actions]).to_rack if blocked_event
             end
 
             request_return
@@ -137,17 +159,19 @@ module Datadog
               )
             end
 
-            if processor.ruleset_info
-              span.set_tag('_dd.appsec.event_rules.version', processor.ruleset_info[:version])
+            if processor.diagnostics
+              diagnostics = processor.diagnostics
+
+              span.set_tag('_dd.appsec.event_rules.version', diagnostics['ruleset_version'])
 
               unless @oneshot_tags_sent
                 # Small race condition, but it's inoccuous: worst case the tags
                 # are sent a couple of times more than expected
                 @oneshot_tags_sent = true
 
-                span.set_tag('_dd.appsec.event_rules.loaded', processor.ruleset_info[:loaded].to_f)
-                span.set_tag('_dd.appsec.event_rules.error_count', processor.ruleset_info[:failed].to_f)
-                span.set_tag('_dd.appsec.event_rules.errors', JSON.dump(processor.ruleset_info[:errors]))
+                span.set_tag('_dd.appsec.event_rules.loaded', diagnostics['rules']['loaded'].size.to_f)
+                span.set_tag('_dd.appsec.event_rules.error_count', diagnostics['rules']['failed'].size.to_f)
+                span.set_tag('_dd.appsec.event_rules.errors', JSON.dump(diagnostics['rules']['errors']))
                 span.set_tag('_dd.appsec.event_rules.addresses', JSON.dump(processor.addresses))
 
                 # Ensure these tags reach the backend

data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb

@@ -20,11 +20,12 @@ module Datadog
               def watch_request_action(gateway = Instrumentation.gateway)
                 gateway.watch('rails.request.action', :appsec) do |stack, gateway_request|
                   block = false
+
                   event = nil
                   scope = gateway_request.env[Datadog::AppSec::Ext::SCOPE_KEY]
 
                   AppSec::Reactive::Operation.new('rails.request.action') do |op|
-                    Rails::Reactive::Action.subscribe(op, scope.processor_context) do |result, _block|
+                    Rails::Reactive::Action.subscribe(op, scope.processor_context) do |result|
                       if result.status == :match
                         # TODO: should this hash be an Event instance instead?
                         event = {
@@ -44,7 +45,7 @@ module Datadog
                       end
                     end
 
-                    _result, block = Rails::Reactive::Action.publish(op, gateway_request)
+                    block = Rails::Reactive::Action.publish(op, gateway_request)
                   end
 
                   next [nil, [[:block, event]]] if block

data/lib/datadog/appsec/contrib/rails/patcher.rb

@@ -83,9 +83,15 @@ module Datadog
                 super
               end
 
-              if request_response && request_response.any? { |action, _event| action == :block }
-                @_response = AppSec::Response.negotiate(env).to_action_dispatch_response
-                request_return = @_response.body
+              if request_response
+                blocked_event = request_response.find { |action, _options| action == :block }
+                if blocked_event
+                  @_response = AppSec::Response.negotiate(
+                    env,
+                    blocked_event.last[:actions]
+                  ).to_action_dispatch_response
+                  request_return = @_response.body
+                end
               end
 
               request_return

data/lib/datadog/appsec/contrib/rails/reactive/action.rb

@@ -45,11 +45,8 @@ module Datadog
                 when :match
                   Datadog.logger.debug { "WAF: #{result.inspect}" }
 
-                  block = result.actions.include?('block')
-
-                  yield [result, block]
-
-                  throw(:block, [result, true]) if block
+                  yield result
+                  throw(:block, true) unless result.actions.empty?
                 when :ok
                   Datadog.logger.debug { "WAF OK: #{result.inspect}" }
                 when :invalid_call

data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb

@@ -22,11 +22,12 @@ module Datadog
               def watch_request_dispatch(gateway = Instrumentation.gateway)
                 gateway.watch('sinatra.request.dispatch', :appsec) do |stack, gateway_request|
                   block = false
+
                   event = nil
                   scope = gateway_request.env[Datadog::AppSec::Ext::SCOPE_KEY]
 
                   AppSec::Reactive::Operation.new('sinatra.request.dispatch') do |op|
-                    Rack::Reactive::RequestBody.subscribe(op, scope.processor_context) do |result, _block|
+                    Rack::Reactive::RequestBody.subscribe(op, scope.processor_context) do |result|
                       if result.status == :match
                         # TODO: should this hash be an Event instance instead?
                         event = {
@@ -46,7 +47,7 @@ module Datadog
                       end
                     end
 
-                    _result, block = Rack::Reactive::RequestBody.publish(op, gateway_request)
+                    block = Rack::Reactive::RequestBody.publish(op, gateway_request)
                   end
 
                   next [nil, [[:block, event]]] if block
@@ -65,11 +66,12 @@ module Datadog
               def watch_request_routed(gateway = Instrumentation.gateway)
                 gateway.watch('sinatra.request.routed', :appsec) do |stack, (gateway_request, gateway_route_params)|
                   block = false
+
                   event = nil
                   scope = gateway_request.env[Datadog::AppSec::Ext::SCOPE_KEY]
 
                   AppSec::Reactive::Operation.new('sinatra.request.routed') do |op|
-                    Sinatra::Reactive::Routed.subscribe(op, scope.processor_context) do |result, _block|
+                    Sinatra::Reactive::Routed.subscribe(op, scope.processor_context) do |result|
                       if result.status == :match
                         # TODO: should this hash be an Event instance instead?
                         event = {
@@ -89,7 +91,7 @@ module Datadog
                       end
                     end
 
-                    _result, block = Sinatra::Reactive::Routed.publish(op, [gateway_request, gateway_route_params])
+                    block = Sinatra::Reactive::Routed.publish(op, [gateway_request, gateway_route_params])
                   end
 
                   next [nil, [[:block, event]]] if block

data/lib/datadog/appsec/contrib/sinatra/patcher.rb

@@ -65,9 +65,12 @@ module Datadog
               catch(Datadog::AppSec::Contrib::Sinatra::Ext::ROUTE_INTERRUPT) { super }
             end
 
-            if request_response && request_response.any? { |action, _event| action == :block }
-              self.response = AppSec::Response.negotiate(env).to_sinatra_response
-              request_return = nil
+            if request_response
+              blocked_event = request_response.find { |action, _options| action == :block }
+              if blocked_event
+                self.response = AppSec::Response.negotiate(env, blocked_event.last[:actions]).to_sinatra_response
+                request_return = nil
+              end
             end
 
             request_return
@@ -103,11 +106,14 @@ module Datadog
                 [gateway_request, gateway_route_params]
               )
 
-              if request_response && request_response.any? { |action, _event| action == :block }
-                self.response = AppSec::Response.negotiate(env).to_sinatra_response
+              if request_response
+                blocked_event = request_response.find { |action, _options| action == :block }
+                if blocked_event
+                  self.response = AppSec::Response.negotiate(env, blocked_event.last[:actions]).to_sinatra_response
 
-                # interrupt request and return response to dispatch! for consistency
-                throw(Datadog::AppSec::Contrib::Sinatra::Ext::ROUTE_INTERRUPT, response)
+                  # interrupt request and return response to dispatch! for consistency
+                  throw(Datadog::AppSec::Contrib::Sinatra::Ext::ROUTE_INTERRUPT, response)
+                end
               end
 
               yield(*args)

data/lib/datadog/appsec/contrib/sinatra/reactive/routed.rb

@@ -40,11 +40,8 @@ module Datadog
                 when :match
                   Datadog.logger.debug { "WAF: #{result.inspect}" }
 
-                  block = result.actions.include?('block')
-
-                  yield [result, block]
-
-                  throw(:block, [result, true]) if block
+                  yield result
+                  throw(:block, true) unless result.actions.empty?
                 when :ok
                   Datadog.logger.debug { "WAF OK: #{result.inspect}" }
                 when :invalid_call