ddtrace 0.54.1 → 1.0.0.beta2

data/lib/datadog/appsec/contrib/sinatra/patcher.rb

@@ -0,0 +1,77 @@
+# typed: ignore
+
+require 'datadog/tracing/contrib/rack/middlewares'
+
+require 'datadog/appsec/contrib/patcher'
+require 'datadog/appsec/contrib/sinatra/integration'
+require 'datadog/appsec/contrib/rack/request_middleware'
+require 'datadog/appsec/contrib/sinatra/framework'
+require 'datadog/tracing/contrib/sinatra/framework'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Sinatra
+        # Set tracer configuration at a late enough time
+        module AppSecSetupPatch
+          def setup_middleware(*args, &block)
+            super.tap do
+              Datadog::AppSec::Contrib::Sinatra::Framework.setup
+            end
+          end
+        end
+
+        # Hook into builder before the middleware list gets frozen
+        module DefaultMiddlewarePatch
+          def setup_middleware(*args, &block)
+            builder = args.first
+
+            super.tap do
+              tracing_sinatra_framework = Datadog::Tracing::Contrib::Sinatra::Framework
+              tracing_middleware = Datadog::Tracing::Contrib::Rack::TraceMiddleware
+
+              if tracing_sinatra_framework.include_middleware?(tracing_middleware, builder)
+                tracing_sinatra_framework.add_middleware_after(tracing_middleware,
+                                                               Datadog::AppSec::Contrib::Rack::RequestMiddleware,
+                                                               builder)
+              else
+                tracing_sinatra_framework.add_middleware(Datadog::AppSec::Contrib::Rack::RequestMiddleware, builder)
+              end
+              tracing_sinatra_framework.inspect_middlewares(builder)
+            end
+          end
+        end
+
+        # Patcher for AppSec on Sinatra
+        module Patcher
+          include Datadog::AppSec::Contrib::Patcher
+
+          module_function
+
+          def patched?
+            Patcher.instance_variable_get(:@patched)
+          end
+
+          def target_version
+            Integration.version
+          end
+
+          def patch
+            patch_default_middlewares
+            setup_security
+
+            Patcher.instance_variable_set(:@patched, true)
+          end
+
+          def setup_security
+            ::Sinatra::Base.singleton_class.prepend(AppSecSetupPatch)
+          end
+
+          def patch_default_middlewares
+            ::Sinatra::Base.singleton_class.prepend(DefaultMiddlewarePatch)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/sinatra/request_middleware.rb

@@ -0,0 +1,20 @@
+# typed: true
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Sinatra
+        # Rack middleware for AppSec on Sinatra
+        class RequestMiddleware
+          def initialize(app, opt = {})
+            @app = app
+          end
+
+          def call(env)
+            @app.call(env)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/event.rb

@@ -0,0 +1,106 @@
+# typed: false
+
+require 'datadog/appsec/contrib/rack/request'
+require 'datadog/appsec/contrib/rack/response'
+require 'datadog/appsec/rate_limiter'
+
+module Datadog
+  module AppSec
+    # AppSec event
+    module Event
+      ALLOWED_REQUEST_HEADERS = %w[
+        X-Forwarded-For
+        X-Client-IP
+        X-Real-IP
+        X-Forwarded
+        X-Cluster-Client-IP
+        Forwarded-For
+        Forwarded
+        Via
+        True-Client-IP
+        Content-Length
+        Content-Type
+        Content-Encoding
+        Content-Language
+        Host
+        User-Agent
+        Accept
+        Accept-Encoding
+        Accept-Language
+      ].map!(&:downcase).freeze
+
+      ALLOWED_RESPONSE_HEADERS = %w[
+        Content-Length
+        Content-Type
+        Content-Encoding
+        Content-Language
+      ].map!(&:downcase).freeze
+
+      def self.record(*events)
+        Datadog::AppSec::RateLimiter.limit(:traces) do
+          record_via_span(*events)
+        end
+      end
+
+      # rubocop:disable Metrics/AbcSize
+      def self.record_via_span(*events)
+        events.group_by { |e| e[:trace] }.each do |trace, event_group|
+          unless trace
+            Datadog.logger.debug { "{ error: 'no trace: cannot record', event_group: #{event_group.inspect}}" }
+            next
+          end
+
+          trace.keep!
+
+          # prepare and gather tags to apply
+          trace_tags = event_group.each_with_object({}) do |event, tags|
+            span = event[:span]
+
+            span.set_tag('appsec.event', 'true') if span
+
+            request = event[:request]
+            response = event[:response]
+
+            # TODO: assume HTTP request context for now
+            request_headers = AppSec::Contrib::Rack::Request.headers(request).select do |k, _|
+              ALLOWED_REQUEST_HEADERS.include?(k.downcase)
+            end
+            response_headers = AppSec::Contrib::Rack::Response.headers(response).select do |k, _|
+              ALLOWED_RESPONSE_HEADERS.include?(k.downcase)
+            end
+
+            request_headers.each do |header, value|
+              tags["http.request.headers.#{header}"] = value
+            end
+
+            response_headers.each do |header, value|
+              tags["http.response.headers.#{header}"] = value
+            end
+
+            tags['http.host'] = request.host
+            tags['http.useragent'] = request.user_agent
+            tags['network.client.ip'] = request.ip
+
+            # tags['actor.ip'] = request.ip # TODO: uses client IP resolution algorithm
+            tags['_dd.origin'] = 'appsec'
+
+            # accumulate triggers
+            tags['_dd.appsec.triggers'] ||= []
+            tags['_dd.appsec.triggers'] += event[:waf_result].data
+          end
+
+          # apply tags to root span
+
+          # complex types are unsupported, we need to serialize to a string
+          triggers = trace_tags.delete('_dd.appsec.triggers')
+          trace.set_tag('_dd.appsec.json', JSON.dump({ triggers: triggers }))
+
+          trace_tags.each do |key, value|
+            trace.set_tag(key, value)
+          end
+        end
+      end
+      # rubocop:enable Metrics/AbcSize
+    end
+  end
+end

data/lib/datadog/appsec/extensions.rb

@@ -0,0 +1,121 @@
+# typed: false
+
+require 'forwardable'
+require 'datadog/appsec/configuration'
+
+module Datadog
+  module AppSec
+    # Extends Datadog tracing with AppSec features
+    module Extensions
+      # Inject AppSec into global objects.
+      def self.activate!
+        Core::Configuration::Settings.include(Settings)
+      end
+
+      # Global Datadog configuration mixin
+      module Settings
+        # Exposes AppSec settings through the
+        # `Datadog.configure {|c| c.appsec._option_ }`
+        # configuration path.
+        def appsec
+          @appsec ||= AppSecAdapter.new(AppSec.settings)
+        end
+      end
+
+      # Merges {Datadog::AppSec::Configuration::Settings} and {Datadog::AppSec::Configuration::DSL}
+      # into a single read/write object.
+      class AppSecAdapter
+        extend Forwardable
+
+        def initialize(settings)
+          @settings = settings
+        end
+
+        # Writer methods
+        def trace_rate_limit=(arg)
+          dsl = AppSec::Configuration::DSL.new
+          dsl.trace_rate_limit = arg
+          @settings.merge(dsl)
+        end
+
+        def options(arg)
+          dsl = AppSec::Configuration::DSL.new
+          dsl.options arg
+          @settings.merge(dsl)
+        end
+
+        def instruments(arg)
+          dsl = AppSec::Configuration::DSL.new
+          dsl.instruments arg
+          @settings.merge(dsl)
+        end
+
+        def ruleset=(arg)
+          dsl = AppSec::Configuration::DSL.new
+          dsl.ruleset = arg
+          @settings.merge(dsl)
+        end
+
+        def instrument(*args)
+          dsl = AppSec::Configuration::DSL.new
+          dsl.instrument(*args)
+          @settings.merge(dsl)
+        end
+
+        def waf_timeout=(arg)
+          dsl = AppSec::Configuration::DSL.new
+          dsl.waf_timeout = arg
+          @settings.merge(dsl)
+        end
+
+        def enabled=(arg)
+          dsl = AppSec::Configuration::DSL.new
+          dsl.enabled = arg
+          @settings.merge(dsl)
+        end
+
+        def waf_debug=(arg)
+          dsl = AppSec::Configuration::DSL.new
+          dsl.waf_debug = arg
+          @settings.merge(dsl)
+        end
+
+        # Reader methods
+        def [](arg)
+          @settings[arg]
+        end
+
+        def ruleset
+          @settings.ruleset
+        end
+
+        def waf_timeout
+          @settings.waf_timeout
+        end
+
+        def enabled
+          @settings.enabled
+        end
+
+        def waf_debug
+          @settings.waf_debug
+        end
+
+        def trace_rate_limit
+          @settings.trace_rate_limit
+        end
+
+        def merge(arg)
+          @settings.merge(arg)
+        end
+
+        private
+
+        # Restore to original state, for testing only.
+        def reset!
+          @settings.send(:reset!)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/instrumentation/gateway.rb

@@ -0,0 +1,44 @@
+# typed: true
+
+module Datadog
+  module AppSec
+    # Instrumentation for AppSec
+    module Instrumentation
+      # Instrumentation gateway implementation
+      class Gateway
+        def initialize
+          @middlewares = Hash.new { |h, k| h[k] = [] }
+        end
+
+        def push(name, env, &block)
+          block ||= -> {}
+
+          middlewares = @middlewares[name]
+
+          return [block.call, nil] if middlewares.empty?
+
+          wrapped = lambda do |_env|
+            [block.call, nil]
+          end
+
+          # TODO: handle exceptions, except for wrapped
+          stack = middlewares.reverse.reduce(wrapped) do |next_, middleware|
+            lambda do |env_|
+              middleware.call(next_, env_)
+            end
+          end
+
+          stack.call(env)
+        end
+
+        def watch(name, &block)
+          @middlewares[name] << block
+        end
+      end
+
+      def self.gateway
+        @gateway ||= Gateway.new # TODO: not thread safe
+      end
+    end
+  end
+end

data/lib/datadog/appsec/processor.rb

@@ -0,0 +1,121 @@
+# typed: ignore
+
+require 'datadog/appsec/assets'
+
+module Datadog
+  module AppSec
+    # Processor integrates libddwaf into datadog/appsec
+    class Processor
+      # Interface object to check using case .. when
+      module IOLike
+        def read; end
+        def rewind; end
+
+        def self.===(other)
+          instance_methods.all? { |meth| other.respond_to?(meth) }
+        end
+      end
+
+      def initialize
+        @ruleset = nil
+        @handle = nil
+
+        unless load_libddwaf && load_ruleset && create_waf_handle
+          Datadog.logger.warn { 'AppSec is disabled, see logged errors above' }
+        end
+      end
+
+      def ready?
+        !@ruleset.nil? && !@handle.nil?
+      end
+
+      def new_context
+        Datadog::AppSec::WAF::Context.new(@handle)
+      end
+
+      private
+
+      def load_libddwaf
+        Processor.require_libddwaf && Processor.libddwaf_provides_waf?
+      end
+
+      def load_ruleset
+        ruleset_setting = Datadog::AppSec.settings.ruleset
+
+        begin
+          @ruleset = case ruleset_setting
+                     when :recommended, :risky, :strict
+                       JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset_setting))
+                     when String
+                       JSON.parse(File.read(ruleset_setting))
+                     when IOLike
+                       JSON.parse(ruleset_setting.read).tap { ruleset_setting.rewind }
+                     when Hash
+                       ruleset_setting
+                     else
+                       raise ArgumentError, "unsupported value for ruleset setting: #{ruleset_setting.inspect}"
+                     end
+
+          true
+        rescue StandardError => e
+          Datadog.logger.error do
+            "libddwaf ruleset failed to load, ruleset: #{ruleset_setting.inspect} error: #{e.inspect}"
+          end
+
+          false
+        end
+      end
+
+      def create_waf_handle
+        # TODO: this may need to be reset if the main Datadog logging level changes after initialization
+        Datadog::AppSec::WAF.logger = Datadog.logger if Datadog.logger.debug? && Datadog::AppSec.settings.waf_debug
+
+        @handle = Datadog::AppSec::WAF::Handle.new(@ruleset)
+
+        true
+      rescue StandardError => e
+        Datadog.logger.error do
+          "libddwaf failed to initialize, error: #{e.inspect}"
+        end
+
+        false
+      end
+
+      class << self
+        # check whether libddwaf is required *and* able to provide the needed feature
+        def libddwaf_provides_waf?
+          defined?(Datadog::AppSec::WAF) ? true : false
+        end
+
+        # libddwaf raises a LoadError on unsupported platforms; it may at some
+        # point succeed in being required yet not provide a specific needed feature.
+        def require_libddwaf
+          Datadog.logger.debug { "libddwaf platform: #{libddwaf_platform}" }
+
+          require 'libddwaf'
+
+          true
+        rescue LoadError => e
+          Datadog.logger.error do
+            'libddwaf failed to load,' \
+              "installed platform: #{libddwaf_platform} ruby platforms: #{ruby_platforms} error: #{e.inspect}"
+          end
+
+          false
+        end
+
+        def libddwaf_spec
+          Gem.loaded_specs['libddwaf']
+        end
+
+        def libddwaf_platform
+          libddwaf_spec ? libddwaf_spec.platform.to_s : 'unknown'
+        end
+
+        def ruby_platforms
+          Gem.platforms.map(&:to_s)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/rate_limiter.rb

@@ -0,0 +1,55 @@
+# typed: true
+
+module Datadog
+  module AppSec
+    # Simple per-thread rate limiter
+    # Since AppSec marks sampling to keep on a security event, this limits the flood of egress traces involving AppSec
+    class RateLimiter
+      def initialize(rate)
+        @rate = rate
+        @timestamps = []
+      end
+
+      def limit
+        now = Time.now.to_f
+
+        loop do
+          oldest = @timestamps.first
+
+          break if oldest.nil? || now - oldest < 1
+
+          @timestamps.shift
+        end
+
+        @timestamps << now
+
+        if (count = @timestamps.count) <= @rate
+          yield
+        else
+          Datadog.logger.debug { "Rate limit hit: #{count}/#{@rate} AppSec traces/second" }
+        end
+      end
+
+      class << self
+        def limit(name, &block)
+          rate_limiter(name).limit(&block)
+        end
+
+        protected
+
+        def rate_limiter(name)
+          case name
+          when :traces
+            Thread.current[:datadog_security_trace_rate_limiter] ||= RateLimiter.new(trace_rate_limit)
+          else
+            raise "unsupported rate limiter: #{name.inspect}"
+          end
+        end
+
+        def trace_rate_limit
+          Datadog::AppSec.settings.trace_rate_limit
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/reactive/address_hash.rb

@@ -0,0 +1,18 @@
+# typed: true
+
+module Datadog
+  module AppSec
+    module Reactive
+      # Address for Reactive Engine
+      class AddressHash < Hash
+        def addresses
+          keys.flatten
+        end
+
+        def with(address)
+          keys.select { |k| k.include?(address) }
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/reactive/engine.rb

@@ -0,0 +1,44 @@
+# typed: true
+
+require 'datadog/appsec/reactive/address_hash'
+require 'datadog/appsec/reactive/subscriber'
+
+module Datadog
+  module AppSec
+    module Reactive
+      # Reactive Engine
+      class Engine
+        def initialize
+          @data = {}
+          @subscribers = AddressHash.new { |h, k| h[k] = [] } # TODO: move to AddressHash initializer
+          @children = []
+        end
+
+        def subscribe(*addresses, &block)
+          @subscribers[addresses.freeze] << Subscriber.new(&block).freeze # TODO: move freeze to Subscriber
+        end
+
+        def publish(address, data)
+          # check if someone has address subscribed
+          if @subscribers.addresses.include?(address)
+
+            # someone will be interested, set data
+            @data[address] = data
+
+            # find candidates i.e address groups that contain the just posted address
+            @subscribers.with(address).each do |addresses|
+              # find targets to the address group containing the posted address
+              subscribers = @subscribers[addresses]
+
+              # is all data for the targets available?
+              if (addresses - @data.keys).empty?
+                hash = addresses.each_with_object({}) { |a, h| h[a] = @data[a] }
+                subscribers.each { |s| s.call(*hash.values) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/reactive/operation.rb

@@ -0,0 +1,52 @@
+# typed: true
+
+require 'datadog/appsec'
+require 'datadog/appsec/reactive/engine'
+
+module Datadog
+  module AppSec
+    module Reactive
+      # Reactive Engine nested operation tracking
+      class Operation
+        attr_reader :reactive,
+                    :parent,
+                    :name
+
+        def initialize(name, parent = nil, reactive_engine = nil)
+          Datadog.logger.debug { "operation: #{name} initialize" }
+          @name = name
+          @parent = parent
+          @reactive = reactive_engine || parent && parent.reactive || Reactive::Engine.new
+
+          # TODO: concurrent store
+          # TODO: constant
+          Thread.current[:datadog_security_active_operation] = self
+
+          yield self if block_given?
+        ensure
+          finalize
+        end
+
+        # TODO: use structs instead of an arg splat
+        def subscribe(*addresses, &block)
+          reactive.subscribe(*addresses, &block)
+        end
+
+        def publish(address, data)
+          reactive.publish(address, data)
+        end
+
+        def finalize
+          Datadog.logger.debug { "operation: #{name} finalize" }
+          Thread.current[:datadog_security_active_operation] = parent
+        end
+
+        class << self
+          def active
+            Thread.current[:datadog_security_active_operation]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/reactive/subscriber.rb

@@ -0,0 +1,18 @@
+# typed: true
+
+module Datadog
+  module AppSec
+    module Reactive
+      # Reactive Engine subscriber
+      class Subscriber
+        def initialize(&block)
+          @block = block
+        end
+
+        def call(*args)
+          @block.call(*args)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec.rb

@@ -0,0 +1,23 @@
+# typed: false
+
+require 'datadog/appsec/configuration'
+require 'datadog/appsec/extensions'
+
+module Datadog
+  # Namespace for Datadog AppSec instrumentation
+  module AppSec
+    include Configuration
+
+    def self.writer
+      @writer ||= Writer.new
+    end
+
+    # Expose AppSec to global shared objects
+    Extensions.activate!
+  end
+end
+
+# Integrations
+require 'datadog/appsec/contrib/rack/integration'
+require 'datadog/appsec/contrib/sinatra/integration'
+require 'datadog/appsec/contrib/rails/integration'