ddtrace 1.5.0 → 1.5.1

data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb

@@ -16,8 +16,10 @@ module Datadog
           module Watcher
             # rubocop:disable Metrics/AbcSize
             # rubocop:disable Metrics/MethodLength
+            # rubocop:disable Metrics/CyclomaticComplexity
+            # rubocop:disable Metrics/PerceivedComplexity
             def self.watch
-              Instrumentation.gateway.watch('rack.request') do |stack, request|
+              Instrumentation.gateway.watch('rack.request', :appsec) do |stack, request|
                 block = false
                 event = nil
                 waf_context = request.env['datadog.waf.context']
@@ -26,23 +28,24 @@ module Datadog
                   trace = active_trace
                   span = active_span
 
-                  Rack::Reactive::Request.subscribe(op, waf_context) do |action, result, _block|
-                    record = [:block, :monitor].include?(action)
-                    if record
+                  Rack::Reactive::Request.subscribe(op, waf_context) do |result, _block|
+                    if result.status == :match
                       # TODO: should this hash be an Event instance instead?
                       event = {
                         waf_result: result,
                         trace: trace,
                         span: span,
                         request: request,
-                        action: action
+                        actions: result.actions
                       }
 
+                      span.set_tag('appsec.event', 'true') if span
+
                       waf_context.events << event
                     end
                   end
 
-                  _action, _result, block = Rack::Reactive::Request.publish(op, request)
+                  _result, block = Rack::Reactive::Request.publish(op, request)
                 end
 
                 next [nil, [[:block, event]]] if block
@@ -57,7 +60,7 @@ module Datadog
                 [ret, res]
               end
 
-              Instrumentation.gateway.watch('rack.response') do |stack, response|
+              Instrumentation.gateway.watch('rack.response', :appsec) do |stack, response|
                 block = false
                 event = nil
                 waf_context = response.instance_eval { @waf_context }
@@ -66,23 +69,24 @@ module Datadog
                   trace = active_trace
                   span = active_span
 
-                  Rack::Reactive::Response.subscribe(op, waf_context) do |action, result, _block|
-                    record = [:block, :monitor].include?(action)
-                    if record
+                  Rack::Reactive::Response.subscribe(op, waf_context) do |result, _block|
+                    if result.status == :match
                       # TODO: should this hash be an Event instance instead?
                       event = {
                         waf_result: result,
                         trace: trace,
                         span: span,
                         response: response,
-                        action: action
+                        actions: result.actions
                       }
 
+                      span.set_tag('appsec.event', 'true') if span
+
                       waf_context.events << event
                     end
                   end
 
-                  _action, _result, block = Rack::Reactive::Response.publish(op, response)
+                  _result, block = Rack::Reactive::Response.publish(op, response)
                 end
 
                 next [nil, [[:block, event]]] if block
@@ -97,7 +101,7 @@ module Datadog
                 [ret, res]
               end
 
-              Instrumentation.gateway.watch('rack.request.body') do |stack, request|
+              Instrumentation.gateway.watch('rack.request.body', :appsec) do |stack, request|
                 block = false
                 event = nil
                 waf_context = request.env['datadog.waf.context']
@@ -106,23 +110,24 @@ module Datadog
                   trace = active_trace
                   span = active_span
 
-                  Rack::Reactive::RequestBody.subscribe(op, waf_context) do |action, result, _block|
-                    record = [:block, :monitor].include?(action)
-                    if record
+                  Rack::Reactive::RequestBody.subscribe(op, waf_context) do |result, _block|
+                    if result.status == :match
                       # TODO: should this hash be an Event instance instead?
                       event = {
                         waf_result: result,
                         trace: trace,
                         span: span,
                         request: request,
-                        action: action
+                        actions: result.actions
                       }
 
+                      span.set_tag('appsec.event', 'true') if span
+
                       waf_context.events << event
                     end
                   end
 
-                  _action, _result, block = Rack::Reactive::RequestBody.publish(op, request)
+                  _result, block = Rack::Reactive::RequestBody.publish(op, request)
                 end
 
                 next [nil, [[:block, event]]] if block
@@ -137,6 +142,8 @@ module Datadog
                 [ret, res]
               end
             end
+            # rubocop:enable Metrics/CyclomaticComplexity
+            # rubocop:enable Metrics/PerceivedComplexity
             # rubocop:enable Metrics/MethodLength
             # rubocop:enable Metrics/AbcSize
 

data/lib/datadog/appsec/contrib/rack/reactive/request.rb

@@ -54,27 +54,27 @@ module Datadog
                 }
 
                 waf_timeout = Datadog::AppSec.settings.waf_timeout
-                action, result = waf_context.run(waf_args, waf_timeout)
+                result = waf_context.run(waf_args, waf_timeout)
 
                 Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
 
-                # TODO: encapsulate return array in a type
-                case action
-                when :monitor
+                case result.status
+                when :match
                   Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, false]
-                when :block
-                  Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, true]
-                  throw(:block, [action, result, true])
-                when :good
+
+                  block = result.actions.include?('block')
+
+                  yield [result, block]
+
+                  throw(:block, [result, true]) if block
+                when :ok
                   Datadog.logger.debug { "WAF OK: #{result.inspect}" }
                 when :invalid_call
                   Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
                 when :invalid_rule, :invalid_flow, :no_rule
                   Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
                 else
-                  Datadog.logger.debug { "WAF UNKNOWN: #{action.inspect} #{result.inspect}" }
+                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
                 end
               end
             end

data/lib/datadog/appsec/contrib/rack/reactive/request_body.rb

@@ -32,27 +32,27 @@ module Datadog
                 }
 
                 waf_timeout = Datadog::AppSec.settings.waf_timeout
-                action, result = waf_context.run(waf_args, waf_timeout)
+                result = waf_context.run(waf_args, waf_timeout)
 
                 Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
 
-                # TODO: encapsulate return array in a type
-                case action
-                when :monitor
+                case result.status
+                when :match
                   Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, false]
-                when :block
-                  Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, true]
-                  throw(:block, [action, result, true])
-                when :good
+
+                  block = result.actions.include?('block')
+
+                  yield [result, block]
+
+                  throw(:block, [result, true]) if block
+                when :ok
                   Datadog.logger.debug { "WAF OK: #{result.inspect}" }
                 when :invalid_call
                   Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
                 when :invalid_rule, :invalid_flow, :no_rule
                   Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
                 else
-                  Datadog.logger.debug { "WAF UNKNOWN: #{action.inspect} #{result.inspect}" }
+                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
                 end
               end
             end

data/lib/datadog/appsec/contrib/rack/reactive/response.rb

@@ -32,27 +32,27 @@ module Datadog
                 }
 
                 waf_timeout = Datadog::AppSec.settings.waf_timeout
-                action, result = waf_context.run(waf_args, waf_timeout)
+                result = waf_context.run(waf_args, waf_timeout)
 
                 Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
 
-                # TODO: encapsulate return array in a type
-                case action
-                when :monitor
+                case result.status
+                when :match
                   Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, false]
-                when :block
-                  Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, true]
-                  throw(:block, [action, result, true])
-                when :good
+
+                  block = result.actions.include?('block')
+
+                  yield [result, block]
+
+                  throw(:block, [result, true]) if block
+                when :ok
                   Datadog.logger.debug { "WAF OK: #{result.inspect}" }
                 when :invalid_call
                   Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
                 when :invalid_rule, :invalid_flow, :no_rule
                   Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
                 else
-                  Datadog.logger.debug { "WAF UNKNOWN: #{action.inspect} #{result.inspect}" }
+                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
                 end
               end
             end

data/lib/datadog/appsec/contrib/rack/request.rb

@@ -47,6 +47,9 @@ module Datadog
           end
 
           def self.form_hash(request)
+            # force form data processing
+            request.POST if request.form_data?
+
             # usually Hash<String,String> but can be a more complex
             # Hash<String,String||Array||Hash> when e.g coming from JSON
             request.env['rack.request.form_hash']

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -21,7 +21,7 @@ module Datadog
           end
 
           def call(env)
-            return @app.call(env) unless @processor.ready?
+            return @app.call(env) unless Datadog.configuration.appsec.enabled && @processor.ready?
 
             # TODO: handle exceptions, except for @app.call
 
@@ -57,6 +57,7 @@ module Datadog
             request_return
           ensure
             add_waf_runtime_tags(context) if context
+            context.finalize if context
           end
 
           private

data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb

@@ -13,7 +13,7 @@ module Datadog
           # Watcher for Rails gateway events
           module Watcher
             def self.watch
-              Instrumentation.gateway.watch('rails.request.action') do |stack, request|
+              Instrumentation.gateway.watch('rails.request.action', :appsec) do |stack, request|
                 block = false
                 event = nil
                 waf_context = request.env['datadog.waf.context']
@@ -22,23 +22,24 @@ module Datadog
                   trace = active_trace
                   span = active_span
 
-                  Rails::Reactive::Action.subscribe(op, waf_context) do |action, result, _block|
-                    record = [:block, :monitor].include?(action)
-                    if record
+                  Rails::Reactive::Action.subscribe(op, waf_context) do |result, _block|
+                    if result.status == :match
                       # TODO: should this hash be an Event instance instead?
                       event = {
                         waf_result: result,
                         trace: trace,
                         span: span,
                         request: request,
-                        action: action
+                        actions: result.actions
                       }
 
+                      span.set_tag('appsec.event', 'true') if span
+
                       waf_context.events << event
                     end
                   end
 
-                  _action, _result, block = Rails::Reactive::Action.publish(op, request)
+                  _result, block = Rails::Reactive::Action.publish(op, request)
                 end
 
                 next [nil, [[:block, event]]] if block

data/lib/datadog/appsec/contrib/rails/reactive/action.rb

@@ -36,27 +36,27 @@ module Datadog
                 }
 
                 waf_timeout = Datadog::AppSec.settings.waf_timeout
-                action, result = waf_context.run(waf_args, waf_timeout)
+                result = waf_context.run(waf_args, waf_timeout)
 
                 Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
 
-                # TODO: encapsulate return array in a type
-                case action
-                when :monitor
+                case result.status
+                when :match
                   Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, false]
-                when :block
-                  Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, true]
-                  throw(:block, [action, result, true])
-                when :good
+
+                  block = result.actions.include?('block')
+
+                  yield [result, block]
+
+                  throw(:block, [result, true]) if block
+                when :ok
                   Datadog.logger.debug { "WAF OK: #{result.inspect}" }
                 when :invalid_call
                   Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
                 when :invalid_rule, :invalid_flow, :no_rule
                   Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
                 else
-                  Datadog.logger.debug { "WAF UNKNOWN: #{action.inspect} #{result.inspect}" }
+                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
                 end
               end
             end

data/lib/datadog/appsec/contrib/rails/request.rb

@@ -7,6 +7,9 @@ module Datadog
         # Normalized extration of data from ActionDispatch::Request
         module Request
           def self.parsed_body(request)
+            # force body parameter parsing, which is done lazily by Rails
+            request.parameters
+
             # usually Hash<String,String> but can be a more complex
             # Hash<String,String||Array||Hash> when e.g coming from JSON or
             # with Rails advanced param square bracket parsing

data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb

@@ -15,7 +15,7 @@ module Datadog
           module Watcher
             # rubocop:disable Metrics/MethodLength
             def self.watch
-              Instrumentation.gateway.watch('sinatra.request.dispatch') do |stack, request|
+              Instrumentation.gateway.watch('sinatra.request.dispatch', :appsec) do |stack, request|
                 block = false
                 event = nil
                 waf_context = request.env['datadog.waf.context']
@@ -24,23 +24,24 @@ module Datadog
                   trace = active_trace
                   span = active_span
 
-                  Rack::Reactive::RequestBody.subscribe(op, waf_context) do |action, result, _block|
-                    record = [:block, :monitor].include?(action)
-                    if record
+                  Rack::Reactive::RequestBody.subscribe(op, waf_context) do |result, _block|
+                    if result.status == :match
                       # TODO: should this hash be an Event instance instead?
                       event = {
                         waf_result: result,
                         trace: trace,
                         span: span,
                         request: request,
-                        action: action
+                        actions: result.actions
                       }
 
+                      span.set_tag('appsec.event', 'true') if span
+
                       waf_context.events << event
                     end
                   end
 
-                  _action, _result, block = Rack::Reactive::RequestBody.publish(op, request)
+                  _result, block = Rack::Reactive::RequestBody.publish(op, request)
                 end
 
                 next [nil, [[:block, event]]] if block
@@ -55,7 +56,7 @@ module Datadog
                 [ret, res]
               end
 
-              Instrumentation.gateway.watch('sinatra.request.routed') do |stack, (request, route_params)|
+              Instrumentation.gateway.watch('sinatra.request.routed', :appsec) do |stack, (request, route_params)|
                 block = false
                 event = nil
                 waf_context = request.env['datadog.waf.context']
@@ -64,23 +65,24 @@ module Datadog
                   trace = active_trace
                   span = active_span
 
-                  Sinatra::Reactive::Routed.subscribe(op, waf_context) do |action, result, _block|
-                    record = [:block, :monitor].include?(action)
-                    if record
+                  Sinatra::Reactive::Routed.subscribe(op, waf_context) do |result, _block|
+                    if result.status == :match
                       # TODO: should this hash be an Event instance instead?
                       event = {
                         waf_result: result,
                         trace: trace,
                         span: span,
                         request: request,
-                        action: action
+                        actions: result.actions
                       }
 
+                      span.set_tag('appsec.event', 'true') if span
+
                       waf_context.events << event
                     end
                   end
 
-                  _action, _result, block = Sinatra::Reactive::Routed.publish(op, [request, route_params])
+                  _result, block = Sinatra::Reactive::Routed.publish(op, [request, route_params])
                 end
 
                 next [nil, [[:block, event]]] if block

data/lib/datadog/appsec/contrib/sinatra/reactive/routed.rb

@@ -31,27 +31,27 @@ module Datadog
                 }
 
                 waf_timeout = Datadog::AppSec.settings.waf_timeout
-                action, result = waf_context.run(waf_args, waf_timeout)
+                result = waf_context.run(waf_args, waf_timeout)
 
                 Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
 
-                # TODO: encapsulate return array in a type
-                case action
-                when :monitor
+                case result.status
+                when :match
                   Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, false]
-                when :block
-                  Datadog.logger.debug { "WAF: #{result.inspect}" }
-                  yield [action, result, true]
-                  throw(:block, [action, result, true])
-                when :good
+
+                  block = result.actions.include?('block')
+
+                  yield [result, block]
+
+                  throw(:block, [result, true]) if block
+                when :ok
                   Datadog.logger.debug { "WAF OK: #{result.inspect}" }
                 when :invalid_call
                   Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
                 when :invalid_rule, :invalid_flow, :no_rule
                   Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
                 else
-                  Datadog.logger.debug { "WAF UNKNOWN: #{action.inspect} #{result.inspect}" }
+                  Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
                 end
               end
             end

data/lib/datadog/appsec/event.rb

@@ -51,8 +51,6 @@ module Datadog
         end
       end
 
-      # rubocop:disable Metrics/AbcSize
-      # rubocop:disable Metrics/MethodLength
       def self.record_via_span(*events)
         events.group_by { |e| e[:trace] }.each do |trace, event_group|
           unless trace
@@ -64,10 +62,6 @@ module Datadog
 
           # prepare and gather tags to apply
           trace_tags = event_group.each_with_object({}) do |event, tags|
-            span = event[:span]
-
-            span.set_tag('appsec.event', 'true') if span
-
             # TODO: assume HTTP request context for now
 
             if (request = event[:request])
@@ -114,8 +108,6 @@ module Datadog
           end
         end
       end
-      # rubocop:enable Metrics/MethodLength
-      # rubocop:enable Metrics/AbcSize
     end
   end
 end

data/lib/datadog/appsec/instrumentation/gateway.rb

@@ -6,6 +6,20 @@ module Datadog
     module Instrumentation
       # Instrumentation gateway implementation
       class Gateway
+        # Instrumentation gateway middleware
+        class Middleware
+          attr_reader :key, :block
+
+          def initialize(key, &block)
+            @key = key
+            @block = block
+          end
+
+          def call(*args, **kwargs, &block)
+            @block.call(*args, **kwargs, &block)
+          end
+        end
+
         def initialize
           @middlewares = Hash.new { |h, k| h[k] = [] }
         end
@@ -31,8 +45,8 @@ module Datadog
           stack.call(env)
         end
 
-        def watch(name, &block)
-          @middlewares[name] << block
+        def watch(name, key, &block)
+          @middlewares[name] << Middleware.new(key, &block) unless @middlewares[name].any? { |m| m.key == key }
         end
       end
 

data/lib/datadog/appsec/processor.rb

@@ -31,7 +31,7 @@ module Datadog
         def run(*args)
           start_ns = Core::Utils::Time.get_time(:nanosecond)
 
-          ret, res = @context.run(*args)
+          _code, res = @context.run(*args)
 
           stop_ns = Core::Utils::Time.get_time(:nanosecond)
 
@@ -39,7 +39,11 @@ module Datadog
           @time_ext_ns += (stop_ns - start_ns)
           @timeouts += 1 if res.timeout
 
-          [ret, res]
+          res
+        end
+
+        def finalize
+          @context.finalize
         end
       end
 
@@ -64,6 +68,18 @@ module Datadog
         Context.new(self)
       end
 
+      def update_rule_data(data)
+        @handle.update_rule_data(data)
+      end
+
+      def toggle_rules(map)
+        @handle.toggle_rules(map)
+      end
+
+      def finalize
+        @handle.finalize
+      end
+
       protected
 
       attr_reader :handle

data/lib/datadog/core/configuration/settings.rb

@@ -323,8 +323,8 @@ module Datadog
 
             # Parse tags from environment
             env_to_list(Core::Environment::Ext::ENV_TAGS, comma_separated_only: false).each do |tag|
-              pair = tag.split(':')
-              tags[pair.first] = pair.last if pair.length == 2
+              key, value = tag.split(':', 2)
+              tags[key] = value if value && !value.empty?
             end
 
             # Override tags if defined

data/lib/datadog/tracing/contrib/rack/middlewares.rb

@@ -271,7 +271,7 @@ module Datadog
                          request_uri
                        end
 
-            base_url + fullpath
+            ::URI.join(base_url, fullpath).to_s
           end
 
           def parse_user_agent_header(headers)

data/lib/datadog/tracing/contrib/utils/quantization/http.rb

@@ -14,20 +14,28 @@ module Datadog
 
             PLACEHOLDER = '?'.freeze
 
+            # taken from Ruby https://github.com/ruby/uri/blob/ffbab83de6d8748c9454414e02db5317609166eb/lib/uri/rfc3986_parser.rb
+            # but adjusted to parse only <scheme>://<host>:<port>/ components
+            # and stop there, since we don't care about the path, query string,
+            # and fragment components
+            RFC3986_URL_BASE = /\A(?<URI>(?<scheme>[A-Za-z][+\-.0-9A-Za-z]*):(?<hier-part>\/\/(?<authority>(?:(?<userinfo>(?:%\h\h|[!$&-.0-;=A-Z_a-z~])*)@)?(?<host>(?<IP-literal>\[(?:(?<IPv6address>(?:\h{1,4}:){6}(?<ls32>\h{1,4}:\h{1,4}|(?<IPv4address>(?<dec-octet>[1-9]\d|1\d{2}|2[0-4]\d|25[0-5]|\d)\.\g<dec-octet>\.\g<dec-octet>\.\g<dec-octet>))|::(?:\h{1,4}:){5}\g<ls32>|\h{1,4}?::(?:\h{1,4}:){4}\g<ls32>|(?:(?:\h{1,4}:)?\h{1,4})?::(?:\h{1,4}:){3}\g<ls32>|(?:(?:\h{1,4}:){,2}\h{1,4})?::(?:\h{1,4}:){2}\g<ls32>|(?:(?:\h{1,4}:){,3}\h{1,4})?::\h{1,4}:\g<ls32>|(?:(?:\h{1,4}:){,4}\h{1,4})?::\g<ls32>|(?:(?:\h{1,4}:){,5}\h{1,4})?::\h{1,4}|(?:(?:\h{1,4}:){,6}\h{1,4})?::)|(?<IPvFuture>v\h+\.[!$&-.0-;=A-Z_a-z~]+))\])|\g<IPv4address>|(?<reg-name>(?:%\h\h|[!$&-.0-9;=A-Z_a-z~])*))(?::(?<port>\d*))?)))(?:\/|\z)/.freeze # rubocop:disable Style/RegexpLiteral, Layout/LineLength
+
             module_function
 
             def url(url, options = {})
               url!(url, options)
             rescue StandardError
-              options[:placeholder] || PLACEHOLDER
+              placeholder = options[:placeholder] || PLACEHOLDER
+
+              options[:base] == :exclude ? placeholder : "#{base_url(url)}/#{placeholder}"
             end
 
             def base_url(url, options = {})
-              URI.parse(url).tap do |uri|
-                uri.path = ''
-                uri.query = nil
-                uri.fragment = nil
-              end.to_s
+              if (m = RFC3986_URL_BASE.match(url))
+                m[1]
+              else
+                ''
+              end
             end
 
             def url!(url, options = {})

data/lib/ddtrace/transport/traces.rb

@@ -101,6 +101,8 @@ module Datadog
           # Make the trace serializable
           serializable_trace = SerializableTrace.new(trace)
 
+          Datadog.logger.debug { "Flushing trace: #{JSON.dump(serializable_trace)}" }
+
           # Encode the trace
           encoder.encode(serializable_trace)
         end