ddtrace 1.4.2 → 1.19.0

data/lib/datadog/appsec/event.rb

@@ -1,9 +1,7 @@
-# typed: false
-
 require 'json'
+require 'zlib'
+require 'base64'
 
-require_relative 'contrib/rack/request'
-require_relative 'contrib/rack/response'
 require_relative 'rate_limiter'
 
 module Datadog
@@ -38,84 +36,134 @@ module Datadog
         Content-Language
       ].map!(&:downcase).freeze
 
+      MAX_ENCODED_SCHEMA_SIZE = 25000
+      # For more information about this number
+      # please check https://github.com/DataDog/dd-trace-rb/pull/3177#issuecomment-1747221082
+      MIN_SCHEMA_SIZE_FOR_COMPRESSION = 260
+
       # Record events for a trace
       #
       # This is expected to be called only once per trace for the rate limiter
       # to properly apply
-      def self.record(*events)
-        # ensure rate limiter is called only when there are events to record
-        return if events.empty?
+      class << self
+        def record(span, *events)
+          # ensure rate limiter is called only when there are events to record
+          return if events.empty? || span.nil?
 
-        Datadog::AppSec::RateLimiter.limit(:traces) do
-          record_via_span(*events)
+          Datadog::AppSec::RateLimiter.limit(:traces) do
+            record_via_span(span, *events)
+          end
         end
-      end
 
-      # rubocop:disable Metrics/AbcSize
-      # rubocop:disable Metrics/MethodLength
-      def self.record_via_span(*events)
-        events.group_by { |e| e[:trace] }.each do |trace, event_group|
-          unless trace
-            Datadog.logger.debug { "{ error: 'no trace: cannot record', event_group: #{event_group.inspect}}" }
-            next
-          end
+        def record_via_span(span, *events)
+          events.group_by { |e| e[:trace] }.each do |trace, event_group|
+            unless trace
+              Datadog.logger.debug { "{ error: 'no trace: cannot record', event_group: #{event_group.inspect}}" }
+              next
+            end
 
-          trace.keep!
+            trace.keep!
+            trace.set_tag(
+              Datadog::Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
+              Datadog::Tracing::Sampling::Ext::Decision::ASM
+            )
 
-          # prepare and gather tags to apply
-          trace_tags = event_group.each_with_object({}) do |event, tags|
-            span = event[:span]
+            # prepare and gather tags to apply
+            service_entry_tags = build_service_entry_tags(event_group)
 
-            span.set_tag('appsec.event', 'true') if span
+            # apply tags to service entry span
+            service_entry_tags.each do |key, value|
+              span.set_tag(key, value)
+            end
+          end
+        end
 
+        # rubocop:disable Metrics/MethodLength
+        def build_service_entry_tags(event_group)
+          waf_events = []
+          entry_tags = event_group.each_with_object({ '_dd.origin' => 'appsec' }) do |event, tags|
             # TODO: assume HTTP request context for now
-
             if (request = event[:request])
-              request_headers = AppSec::Contrib::Rack::Request.headers(request).select do |k, _|
-                ALLOWED_REQUEST_HEADERS.include?(k.downcase)
-              end
-
-              request_headers.each do |header, value|
-                tags["http.request.headers.#{header}"] = value
+              request.headers.each do |header, value|
+                tags["http.request.headers.#{header}"] = value if ALLOWED_REQUEST_HEADERS.include?(header.downcase)
               end
 
               tags['http.host'] = request.host
               tags['http.useragent'] = request.user_agent
-              tags['network.client.ip'] = request.ip
-
-              # tags['actor.ip'] = request.ip # TODO: uses client IP resolution algorithm
+              tags['network.client.ip'] = request.remote_addr
             end
 
             if (response = event[:response])
-              response_headers = AppSec::Contrib::Rack::Response.headers(response).select do |k, _|
-                ALLOWED_RESPONSE_HEADERS.include?(k.downcase)
+              response.headers.each do |header, value|
+                tags["http.response.headers.#{header}"] = value if ALLOWED_RESPONSE_HEADERS.include?(header.downcase)
               end
+            end
 
-              response_headers.each do |header, value|
-                tags["http.response.headers.#{header}"] = value
+            waf_result = event[:waf_result]
+            # accumulate triggers
+            waf_events += waf_result.events
+
+            waf_result.derivatives.each do |key, value|
+              parsed_value = json_parse(value)
+              next unless parsed_value
+
+              parsed_value_size = parsed_value.size
+
+              schema_value = if parsed_value_size >= MIN_SCHEMA_SIZE_FOR_COMPRESSION
+                               compressed_and_base64_encoded(parsed_value)
+                             else
+                               parsed_value
+                             end
+              next unless schema_value
+
+              if schema_value.size >= MAX_ENCODED_SCHEMA_SIZE
+                Datadog.logger.debug do
+                  "Schema key: #{key} exceeds the max size value. It will not be included as part of the span tags"
+                end
+                next
               end
-            end
 
-            tags['_dd.origin'] = 'appsec'
+              tags[key] = schema_value
+            end
 
-            # accumulate triggers
-            tags['_dd.appsec.triggers'] ||= []
-            tags['_dd.appsec.triggers'] += event[:waf_result].data
+            tags
           end
 
-          # apply tags to root span
+          appsec_events = json_parse({ triggers: waf_events })
+          entry_tags['_dd.appsec.json'] = appsec_events if appsec_events
+          entry_tags
+        end
+        # rubocop:enable Metrics/MethodLength
+
+        private
 
-          # complex types are unsupported, we need to serialize to a string
-          triggers = trace_tags.delete('_dd.appsec.triggers')
-          trace.set_tag('_dd.appsec.json', JSON.dump({ triggers: triggers }))
+        def compressed_and_base64_encoded(value)
+          Base64.encode64(gzip(value))
+        rescue TypeError => e
+          Datadog.logger.debug do
+            "Failed to compress and encode value when populating AppSec::Event. Error: #{e.message}"
+          end
+          nil
+        end
 
-          trace_tags.each do |key, value|
-            trace.set_tag(key, value)
+        def json_parse(value)
+          JSON.dump(value)
+        rescue ArgumentError => e
+          Datadog.logger.debug do
+            "Failed to parse value to JSON when populating AppSec::Event. Error: #{e.message}"
           end
+          nil
+        end
+
+        def gzip(value)
+          sio = StringIO.new
+          # For an in depth comparison of Zlib options check https://github.com/DataDog/dd-trace-rb/pull/3177#issuecomment-1747215473
+          gz = Zlib::GzipWriter.new(sio, Zlib::BEST_SPEED, Zlib::DEFAULT_STRATEGY)
+          gz.write(value)
+          gz.close
+          sio.string
         end
       end
-      # rubocop:enable Metrics/MethodLength
-      # rubocop:enable Metrics/AbcSize
     end
   end
 end

data/lib/datadog/appsec/ext.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Ext
+      INTERRUPT = :datadog_appsec_interrupt
+      SCOPE_KEY = 'datadog.appsec.scope'
+    end
+  end
+end

data/lib/datadog/appsec/extensions.rb

@@ -1,4 +1,4 @@
-# typed: false
+# frozen_string_literal: true
 
 require_relative 'configuration'
 
@@ -8,110 +8,7 @@ module Datadog
     module Extensions
       # Inject AppSec into global objects.
       def self.activate!
-        Core::Configuration::Settings.include(Settings)
-      end
-
-      # Global Datadog configuration mixin
-      module Settings
-        # Exposes AppSec settings through the
-        # `Datadog.configure {|c| c.appsec._option_ }`
-        # configuration path.
-        def appsec
-          @appsec ||= AppSecAdapter.new(AppSec.settings)
-        end
-      end
-
-      # Merges {Datadog::AppSec::Configuration::Settings} and {Datadog::AppSec::Configuration::DSL}
-      # into a single read/write object.
-      class AppSecAdapter
-        def initialize(settings)
-          @settings = settings
-        end
-
-        # Writer methods
-        def trace_rate_limit=(arg)
-          dsl = AppSec::Configuration::DSL.new
-          dsl.trace_rate_limit = arg
-          @settings.merge(dsl)
-        end
-
-        def options(arg)
-          dsl = AppSec::Configuration::DSL.new
-          dsl.options arg
-          @settings.merge(dsl)
-        end
-
-        def instruments(arg)
-          dsl = AppSec::Configuration::DSL.new
-          dsl.instruments arg
-          @settings.merge(dsl)
-        end
-
-        def ruleset=(arg)
-          dsl = AppSec::Configuration::DSL.new
-          dsl.ruleset = arg
-          @settings.merge(dsl)
-        end
-
-        def instrument(*args)
-          dsl = AppSec::Configuration::DSL.new
-          dsl.instrument(*args)
-          @settings.merge(dsl)
-        end
-
-        def waf_timeout=(arg)
-          dsl = AppSec::Configuration::DSL.new
-          dsl.waf_timeout = arg
-          @settings.merge(dsl)
-        end
-
-        def enabled=(arg)
-          dsl = AppSec::Configuration::DSL.new
-          dsl.enabled = arg
-          @settings.merge(dsl)
-        end
-
-        def waf_debug=(arg)
-          dsl = AppSec::Configuration::DSL.new
-          dsl.waf_debug = arg
-          @settings.merge(dsl)
-        end
-
-        # Reader methods
-        def [](arg)
-          @settings[arg]
-        end
-
-        def ruleset
-          @settings.ruleset
-        end
-
-        def waf_timeout
-          @settings.waf_timeout
-        end
-
-        def enabled
-          @settings.enabled
-        end
-
-        def waf_debug
-          @settings.waf_debug
-        end
-
-        def trace_rate_limit
-          @settings.trace_rate_limit
-        end
-
-        def merge(arg)
-          @settings.merge(arg)
-        end
-
-        private
-
-        # Restore to original state, for testing only.
-        def reset!
-          @settings.send(:reset!)
-        end
+        Core::Configuration::Settings.extend(Configuration::Settings)
       end
     end
   end

data/lib/datadog/appsec/instrumentation/gateway/argument.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Instrumentation
+      class Gateway
+        # Base class for Gateway Arguments
+        class Argument; end # rubocop:disable Lint/EmptyClass
+
+        # Gateway User argument
+        class User < Argument
+          attr_reader :id
+
+          def initialize(id)
+            super()
+            @id = id
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/instrumentation/gateway.rb

@@ -1,4 +1,4 @@
-# typed: true
+# frozen_string_literal: true
 
 module Datadog
   module AppSec
@@ -6,6 +6,22 @@ module Datadog
     module Instrumentation
       # Instrumentation gateway implementation
       class Gateway
+        # Instrumentation gateway middleware
+        class Middleware
+          attr_reader :key, :block
+
+          def initialize(key, &block)
+            @key = key
+            @block = block
+          end
+
+          def call(stack, env)
+            @block.call(stack, env)
+          end
+        end
+
+        private_constant :Middleware
+
         def initialize
           @middlewares = Hash.new { |h, k| h[k] = [] }
         end
@@ -13,16 +29,16 @@ module Datadog
         def push(name, env, &block)
           block ||= -> {}
 
-          middlewares = @middlewares[name]
+          middlewares_for_name = middlewares[name]
 
-          return [block.call, nil] if middlewares.empty?
+          return [block.call, nil] if middlewares_for_name.empty?
 
           wrapped = lambda do |_env|
             [block.call, nil]
           end
 
           # TODO: handle exceptions, except for wrapped
-          stack = middlewares.reverse.reduce(wrapped) do |next_, middleware|
+          stack = middlewares_for_name.reverse.reduce(wrapped) do |next_, middleware|
             lambda do |env_|
               middleware.call(next_, env_)
             end
@@ -31,9 +47,13 @@ module Datadog
           stack.call(env)
         end
 
-        def watch(name, &block)
-          @middlewares[name] << block
+        def watch(name, key, &block)
+          @middlewares[name] << Middleware.new(key, &block) unless middlewares[name].any? { |m| m.key == key }
         end
+
+        private
+
+        attr_reader :middlewares
       end
 
       def self.gateway

data/lib/datadog/appsec/instrumentation.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # Instrumentation for AppSec
+    module Instrumentation
+    end
+  end
+end

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require_relative '../../instrumentation/gateway'
+require_relative '../../reactive/operation'
+require_relative '../reactive/set_user'
+
+module Datadog
+  module AppSec
+    module Monitor
+      module Gateway
+        # Watcher for Apssec internal events
+        module Watcher
+          class << self
+            def watch
+              gateway = Instrumentation.gateway
+
+              watch_user_id(gateway)
+            end
+
+            def watch_user_id(gateway = Instrumentation.gateway)
+              gateway.watch('identity.set_user', :appsec) do |stack, user|
+                block = false
+                event = nil
+                scope = Datadog::AppSec.active_scope
+
+                AppSec::Reactive::Operation.new('identity.set_user') do |op|
+                  Monitor::Reactive::SetUser.subscribe(op, scope.processor_context) do |result|
+                    if result.status == :match
+                      # TODO: should this hash be an Event instance instead?
+                      event = {
+                        waf_result: result,
+                        trace: scope.trace,
+                        span: scope.service_entry_span,
+                        user: user,
+                        actions: result.actions
+                      }
+
+                      if scope.service_entry_span
+                        scope.service_entry_span.set_tag('appsec.blocked', 'true') if result.actions.include?('block')
+                        scope.service_entry_span.set_tag('appsec.event', 'true')
+                      end
+
+                      scope.processor_context.events << event
+                    end
+                  end
+
+                  block = Monitor::Reactive::SetUser.publish(op, user)
+                end
+
+                throw(Datadog::AppSec::Ext::INTERRUPT, [nil, [[:block, event]]]) if block
+
+                ret, res = stack.call(user)
+
+                if event
+                  res ||= []
+                  res << [:monitor, event]
+                end
+
+                [ret, res]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/monitor/reactive/set_user.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Monitor
+      module Reactive
+        # Dispatch data from Datadog::Kit::Identity.set_user to the WAF context
+        module SetUser
+          ADDRESSES = [
+            'usr.id',
+          ].freeze
+          private_constant :ADDRESSES
+
+          def self.publish(op, user)
+            catch(:block) do
+              op.publish('usr.id', user.id)
+
+              nil
+            end
+          end
+
+          def self.subscribe(op, waf_context)
+            op.subscribe(*ADDRESSES) do |*values|
+              Datadog.logger.debug { "reacted to #{ADDRESSES.inspect}: #{values.inspect}" }
+
+              user_id = values[0]
+
+              waf_args = {
+                'usr.id' => user_id,
+              }
+
+              waf_timeout = Datadog.configuration.appsec.waf_timeout
+              result = waf_context.run(waf_args, waf_timeout)
+
+              Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
+
+              case result.status
+              when :match
+                Datadog.logger.debug { "WAF: #{result.inspect}" }
+
+                yield result
+                throw(:block, true) unless result.actions.empty?
+              when :ok
+                Datadog.logger.debug { "WAF OK: #{result.inspect}" }
+              when :invalid_call
+                Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
+              when :invalid_rule, :invalid_flow, :no_rule
+                Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
+              else
+                Datadog.logger.debug { "WAF UNKNOWN: #{result.status.inspect} #{result.inspect}" }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/monitor.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require_relative 'monitor/gateway/watcher'
+
+module Datadog
+  module AppSec
+    # Monitor for internal AppSec Events
+    module Monitor
+    end
+  end
+end

data/lib/datadog/appsec/processor/actions.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    class Processor
+      # Actions store the actions information in memory
+      # Also, takes care of merging when RC send new information
+      module Actions
+        class << self
+          def actions
+            @actions ||= []
+          end
+
+          def fecth_configuration(action)
+            actions.find { |action_configuration| action_configuration['id'] == action }
+          end
+
+          def merge(actions_to_merge)
+            return if actions_to_merge.empty?
+
+            if actions.empty?
+              @actions = actions_to_merge
+            else
+              merged_actions = []
+              actions_dup = actions.dup
+
+              actions_to_merge.each do |new_action|
+                existing_action = actions_dup.find { |action| new_action['id'] == action['id'] }
+
+                # the old action is discard and the new kept
+                actions_dup.delete(existing_action) if existing_action
+                merged_actions << new_action
+              end
+
+              @actions = merged_actions.concat(actions_dup)
+            end
+          end
+
+          private
+
+          # Used in tests
+          def reset
+            @actions = []
+          end
+        end
+      end
+    end
+  end
+end