ddtrace 0.54.2 → 1.0.0.beta1

data/lib/datadog/appsec/contrib/sinatra/patcher.rb

@@ -0,0 +1,77 @@
+# typed: ignore
+
+require 'datadog/tracing/contrib/rack/middlewares'
+
+require 'datadog/appsec/contrib/patcher'
+require 'datadog/appsec/contrib/sinatra/integration'
+require 'datadog/appsec/contrib/rack/request_middleware'
+require 'datadog/appsec/contrib/sinatra/framework'
+require 'datadog/tracing/contrib/sinatra/framework'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Sinatra
+        # Set tracer configuration at a late enough time
+        module AppSecSetupPatch
+          def setup_middleware(*args, &block)
+            super.tap do
+              Datadog::AppSec::Contrib::Sinatra::Framework.setup
+            end
+          end
+        end
+
+        # Hook into builder before the middleware list gets frozen
+        module DefaultMiddlewarePatch
+          def setup_middleware(*args, &block)
+            builder = args.first
+
+            super.tap do
+              tracing_sinatra_framework = Datadog::Tracing::Contrib::Sinatra::Framework
+              tracing_middleware = Datadog::Tracing::Contrib::Rack::TraceMiddleware
+
+              if tracing_sinatra_framework.include_middleware?(tracing_middleware, builder)
+                tracing_sinatra_framework.add_middleware_after(tracing_middleware,
+                                                               Datadog::AppSec::Contrib::Rack::RequestMiddleware,
+                                                               builder)
+              else
+                tracing_sinatra_framework.add_middleware(Datadog::AppSec::Contrib::Rack::RequestMiddleware, builder)
+              end
+              tracing_sinatra_framework.inspect_middlewares(builder)
+            end
+          end
+        end
+
+        # Patcher for AppSec on Sinatra
+        module Patcher
+          include Datadog::AppSec::Contrib::Patcher
+
+          module_function
+
+          def patched?
+            Patcher.instance_variable_get(:@patched)
+          end
+
+          def target_version
+            Integration.version
+          end
+
+          def patch
+            patch_default_middlewares
+            setup_security
+
+            Patcher.instance_variable_set(:@patched, true)
+          end
+
+          def setup_security
+            ::Sinatra::Base.singleton_class.prepend(AppSecSetupPatch)
+          end
+
+          def patch_default_middlewares
+            ::Sinatra::Base.singleton_class.prepend(DefaultMiddlewarePatch)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/sinatra/request_middleware.rb

@@ -0,0 +1,18 @@
+module Datadog
+  module AppSec
+    module Contrib
+      module Sinatra
+        # Rack middleware for AppSec on Sinatra
+        class RequestMiddleware
+          def initialize(app, opt = {})
+            @app = app
+          end
+
+          def call(env)
+            @app.call(env)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/event.rb

@@ -0,0 +1,111 @@
+require 'datadog/appsec/contrib/rack/request'
+require 'datadog/appsec/contrib/rack/response'
+require 'datadog/appsec/rate_limiter'
+
+module Datadog
+  module AppSec
+    # AppSec event
+    module Event
+      ALLOWED_REQUEST_HEADERS = %w[
+        X-Forwarded-For
+        X-Client-IP
+        X-Real-IP
+        X-Forwarded
+        X-Cluster-Client-IP
+        Forwarded-For
+        Forwarded
+        Via
+        True-Client-IP
+        Content-Length
+        Content-Type
+        Content-Encoding
+        Content-Language
+        Host
+        User-Agent
+        Accept
+        Accept-Encoding
+        Accept-Language
+      ].map!(&:downcase).freeze
+
+      ALLOWED_RESPONSE_HEADERS = %w[
+        Content-Length
+        Content-Type
+        Content-Encoding
+        Content-Language
+      ].map!(&:downcase).freeze
+
+      def self.record(*events)
+        Datadog::AppSec::RateLimiter.limit(:traces) do
+          record_via_span(*events)
+        end
+      end
+
+      # rubocop:disable Metrics/AbcSize
+      # rubocop:disable Metrics/MethodLength
+      def self.record_via_span(*events)
+        events.group_by { |e| e[:root_span] }.each do |root_span, event_group|
+          unless root_span
+            Datadog.logger.debug { "{ error: 'no root span: cannot record', event_group: #{event_group.inspect}}" }
+            next
+          end
+
+          # TODO: this is a hack but there is no API to do that
+          root_span_tags = root_span.send(:meta).keys
+
+          # prepare and gather tags to apply
+          root_tags = event_group.each_with_object({}) do |event, tags|
+            span = event[:span]
+            trace = event[:trace]
+
+            if span
+              span.set_tag('appsec.event', 'true')
+              trace.keep!
+            end
+
+            request = event[:request]
+            response = event[:response]
+
+            # TODO: assume HTTP request context for now
+            request_headers = AppSec::Contrib::Rack::Request.headers(request).select do |k, _|
+              ALLOWED_REQUEST_HEADERS.include?(k.downcase)
+            end
+            response_headers = AppSec::Contrib::Rack::Response.headers(response).select do |k, _|
+              ALLOWED_RESPONSE_HEADERS.include?(k.downcase)
+            end
+
+            request_headers.each do |header, value|
+              tags["http.request.headers.#{header}"] = value
+            end
+
+            response_headers.each do |header, value|
+              tags["http.response.headers.#{header}"] = value
+            end
+
+            tags['http.host'] = request.host
+            tags['http.useragent'] = request.user_agent
+            tags['network.client.ip'] = request.ip
+
+            # tags['actor.ip'] = request.ip # TODO: uses client IP resolution algorithm
+            tags['_dd.origin'] = 'appsec'
+
+            # accumulate triggers
+            tags['_dd.appsec.triggers'] ||= []
+            tags['_dd.appsec.triggers'] += event[:waf_result].data
+          end
+
+          # apply tags to root span
+
+          # complex types are unsupported, we need to serialize to a string
+          triggers = root_tags.delete('_dd.appsec.triggers')
+          root_span.set_tag('_dd.appsec.json', JSON.dump({ triggers: triggers }))
+
+          root_tags.each do |key, value|
+            root_span.set_tag(key, value.is_a?(String) ? value.encode('UTf-8') : value) unless root_span_tags.include?(key)
+          end
+        end
+      end
+      # rubocop:enable Metrics/AbcSize
+      # rubocop:enable Metrics/MethodLength
+    end
+  end
+end

data/lib/datadog/appsec/extensions.rb

@@ -0,0 +1,120 @@
+# typed: true
+require 'forwardable'
+require 'datadog/appsec/configuration'
+
+module Datadog
+  module AppSec
+    # Extends Datadog tracing with AppSec features
+    module Extensions
+      # Inject AppSec into global objects.
+      def self.activate!
+        Core::Configuration::Settings.include(Settings)
+      end
+
+      # Global Datadog configuration mixin
+      module Settings
+        # Exposes AppSec settings through the
+        # `Datadog.configure {|c| c.appsec._option_ }`
+        # configuration path.
+        def appsec
+          @appsec ||= AppSecAdapter.new(AppSec.settings)
+        end
+      end
+
+      # Merges {Datadog::AppSec::Configuration::Settings} and {Datadog::AppSec::Configuration::DSL}
+      # into a single read/write object.
+      class AppSecAdapter
+        extend Forwardable
+
+        def initialize(settings)
+          @settings = settings
+        end
+
+        # Writer methods
+        def trace_rate_limit=(arg)
+          dsl = AppSec::Configuration::DSL.new
+          dsl.trace_rate_limit = arg
+          @settings.merge(dsl)
+        end
+
+        def options(arg)
+          dsl = AppSec::Configuration::DSL.new
+          dsl.options arg
+          @settings.merge(dsl)
+        end
+
+        def instruments(arg)
+          dsl = AppSec::Configuration::DSL.new
+          dsl.instruments arg
+          @settings.merge(dsl)
+        end
+
+        def ruleset=(arg)
+          dsl = AppSec::Configuration::DSL.new
+          dsl.ruleset = arg
+          @settings.merge(dsl)
+        end
+
+        def instrument(*args)
+          dsl = AppSec::Configuration::DSL.new
+          dsl.instrument(*args)
+          @settings.merge(dsl)
+        end
+
+        def waf_timeout=(arg)
+          dsl = AppSec::Configuration::DSL.new
+          dsl.waf_timeout = arg
+          @settings.merge(dsl)
+        end
+
+        def enabled=(arg)
+          dsl = AppSec::Configuration::DSL.new
+          dsl.enabled = arg
+          @settings.merge(dsl)
+        end
+
+        def waf_debug=(arg)
+          dsl = AppSec::Configuration::DSL.new
+          dsl.waf_debug = arg
+          @settings.merge(dsl)
+        end
+
+        # Reader methods
+        def [](arg)
+          @settings[arg]
+        end
+
+        def ruleset
+          @settings.ruleset
+        end
+
+        def waf_timeout
+          @settings.waf_timeout
+        end
+
+        def enabled
+          @settings.enabled
+        end
+
+        def waf_debug
+          @settings.waf_debug
+        end
+
+        def trace_rate_limit
+          @settings.trace_rate_limit
+        end
+
+        def merge(arg)
+          @settings.merge(arg)
+        end
+
+        private
+
+        # Restore to original state, for testing only.
+        def reset!
+          @settings.send(:reset!)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/instrumentation/gateway.rb

@@ -0,0 +1,42 @@
+module Datadog
+  module AppSec
+    # Instrumentation for AppSec
+    module Instrumentation
+      # Instrumentation gateway implementation
+      class Gateway
+        def initialize
+          @middlewares = Hash.new { |h, k| h[k] = [] }
+        end
+
+        def push(name, env, &block)
+          block ||= -> {}
+
+          middlewares = @middlewares[name]
+
+          return [block.call, nil] if middlewares.empty?
+
+          wrapped = lambda do |_env|
+            [block.call, nil]
+          end
+
+          # TODO: handle exceptions, except for wrapped
+          stack = middlewares.reverse.reduce(wrapped) do |next_, middleware|
+            lambda do |env_|
+              middleware.call(next_, env_)
+            end
+          end
+
+          stack.call(env)
+        end
+
+        def watch(name, &block)
+          @middlewares[name] << block
+        end
+      end
+
+      def self.gateway
+        @gateway ||= Gateway.new # TODO: not thread safe
+      end
+    end
+  end
+end

data/lib/datadog/appsec/rate_limiter.rb

@@ -0,0 +1,53 @@
+module Datadog
+  module AppSec
+    # Simple per-thread rate limiter
+    # Since AppSec marks sampling to keep on a security event, this limits the flood of egress traces involving AppSec
+    class RateLimiter
+      def initialize(rate)
+        @rate = rate
+        @timestamps = []
+      end
+
+      def limit
+        now = Time.now.to_f
+
+        loop do
+          oldest = @timestamps.first
+
+          break if oldest.nil? || now - oldest < 1
+
+          @timestamps.shift
+        end
+
+        @timestamps << now
+
+        if (count = @timestamps.count) <= @rate
+          yield
+        else
+          Datadog.logger.debug { "Rate limit hit: #{count}/#{@rate} AppSec traces/second" }
+        end
+      end
+
+      class << self
+        def limit(name, &block)
+          rate_limiter(name).limit(&block)
+        end
+
+        protected
+
+        def rate_limiter(name)
+          case name
+          when :traces
+            Thread.current[:datadog_security_trace_rate_limiter] ||= RateLimiter.new(trace_rate_limit)
+          else
+            raise "unsupported rate limiter: #{name.inspect}"
+          end
+        end
+
+        def trace_rate_limit
+          Datadog::AppSec.settings.trace_rate_limit
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/reactive/address_hash.rb

@@ -0,0 +1,16 @@
+module Datadog
+  module AppSec
+    module Reactive
+      # Address for Reactive Engine
+      class AddressHash < Hash
+        def addresses
+          keys.flatten
+        end
+
+        def with(address)
+          keys.select { |k| k.include?(address) }
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/reactive/engine.rb

@@ -0,0 +1,42 @@
+require 'datadog/appsec/reactive/address_hash'
+require 'datadog/appsec/reactive/subscriber'
+
+module Datadog
+  module AppSec
+    module Reactive
+      # Reactive Engine
+      class Engine
+        def initialize
+          @data = {}
+          @subscribers = AddressHash.new { |h, k| h[k] = [] } # TODO: move to AddressHash initializer
+          @children = []
+        end
+
+        def subscribe(*addresses, &block)
+          @subscribers[addresses.freeze] << Subscriber.new(&block).freeze # TODO: move freeze to Subscriber
+        end
+
+        def publish(address, data)
+          # check if someone has address subscribed
+          if @subscribers.addresses.include?(address)
+
+            # someone will be interested, set data
+            @data[address] = data
+
+            # find candidates i.e address groups that contain the just posted address
+            @subscribers.with(address).each do |addresses|
+              # find targets to the address group containing the posted address
+              subscribers = @subscribers[addresses]
+
+              # is all data for the targets available?
+              if (addresses - @data.keys).empty?
+                hash = addresses.each_with_object({}) { |a, h| h[a] = @data[a] }
+                subscribers.each { |s| s.call(*hash.values) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/reactive/operation.rb

@@ -0,0 +1,50 @@
+require 'datadog/appsec'
+require 'datadog/appsec/reactive/engine'
+
+module Datadog
+  module AppSec
+    module Reactive
+      # Reactive Engine nested operation tracking
+      class Operation
+        attr_reader :reactive,
+                    :parent,
+                    :name
+
+        def initialize(name, parent = nil, reactive_engine = nil)
+          Datadog.logger.debug { "operation: #{name} initialize" }
+          @name = name
+          @parent = parent
+          @reactive = reactive_engine || parent && parent.reactive || Reactive::Engine.new
+
+          # TODO: concurrent store
+          # TODO: constant
+          Thread.current[:datadog_security_active_operation] = self
+
+          yield self if block_given?
+        ensure
+          finalize
+        end
+
+        # TODO: use structs instead of an arg splat
+        def subscribe(*addresses, &block)
+          reactive.subscribe(*addresses, &block)
+        end
+
+        def publish(address, data)
+          reactive.publish(address, data)
+        end
+
+        def finalize
+          Datadog.logger.debug { "operation: #{name} finalize" }
+          Thread.current[:datadog_security_active_operation] = parent
+        end
+
+        class << self
+          def active
+            Thread.current[:datadog_security_active_operation]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/reactive/subscriber.rb

@@ -0,0 +1,16 @@
+module Datadog
+  module AppSec
+    module Reactive
+      # Reactive Engine subscriber
+      class Subscriber
+        def initialize(&block)
+          @block = block
+        end
+
+        def call(*args)
+          @block.call(*args)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec.rb

@@ -0,0 +1,21 @@
+require 'datadog/appsec/configuration'
+require 'datadog/appsec/extensions'
+
+module Datadog
+  # Namespace for Datadog AppSec instrumentation
+  module AppSec
+    include Configuration
+
+    def self.writer
+      @writer ||= Writer.new
+    end
+
+    # Expose AppSec to global shared objects
+    Extensions.activate!
+  end
+end
+
+# Integrations
+require 'datadog/appsec/contrib/rack/integration'
+require 'datadog/appsec/contrib/sinatra/integration'
+require 'datadog/appsec/contrib/rails/integration'

data/lib/datadog/ci/configuration/components.rb

@@ -1,5 +1,5 @@
 # typed: true
-require 'datadog/ci/context_flush'
+require 'datadog/ci/flush'
 
 module Datadog
   module CI
@@ -8,22 +8,22 @@ module Datadog
       module Components
         def initialize(settings)
           # Activate CI mode if enabled
-          activate_ci_mode!(settings) if settings.ci_mode.enabled
+          activate_ci!(settings) if settings.ci.enabled
 
           # Initialize normally
           super
         end
 
-        def activate_ci_mode!(settings)
+        def activate_ci!(settings)
           # Activate underlying tracing test mode
-          settings.test_mode.enabled = true
+          settings.tracing.test_mode.enabled = true
 
-          # Choose user defined ContextFlush or default to CI ContextFlush
-          settings.test_mode.context_flush = settings.ci_mode.context_flush \
-                                             || Datadog::CI::ContextFlush::Finished.new
+          # Choose user defined TraceFlush or default to CI TraceFlush
+          settings.tracing.test_mode.trace_flush = settings.ci.trace_flush \
+                                             || CI::Flush::Finished.new
 
           # Pass through any other options
-          settings.test_mode.writer_options = settings.ci_mode.writer_options
+          settings.tracing.test_mode.writer_options = settings.ci.writer_options
         end
       end
     end

data/lib/datadog/ci/configuration/settings.rb

@@ -13,13 +13,22 @@ module Datadog
 
         def self.add_settings!(base)
           base.class_eval do
-            settings :ci_mode do
+            settings :ci do
               option :enabled do |o|
-                o.default { env_to_bool(Datadog::CI::Ext::Settings::ENV_MODE_ENABLED, false) }
+                o.default { env_to_bool(CI::Ext::Settings::ENV_MODE_ENABLED, false) }
                 o.lazy
               end
 
-              option :context_flush do |o|
+              # DEV: Alias to Datadog::Tracing::Contrib::Extensions::Configuration::Settings#instrument.
+              # DEV: Should be removed when `c.ci.instrument` namespacing is complete.
+              define_method(:instrument) do |integration_name, options = {}, &block|
+                Datadog.configuration.send(:instrument, integration_name, options, &block)
+              end
+
+              # TODO: Deprecate in the next major version, as `instrument` better describes this method's purpose
+              alias_method :use, :instrument
+
+              option :trace_flush do |o|
                 o.default { nil }
                 o.lazy
               end

data/lib/datadog/ci/contrib/cucumber/configuration/settings.rb

@@ -1,5 +1,5 @@
 # typed: false
-require 'ddtrace/contrib/configuration/settings'
+require 'datadog/tracing/contrib/configuration/settings'
 require 'datadog/ci/contrib/cucumber/ext'
 
 module Datadog
@@ -8,7 +8,8 @@ module Datadog
       module Cucumber
         module Configuration
           # Custom settings for the Cucumber integration
-          class Settings < Datadog::Contrib::Configuration::Settings
+          # TODO: mark as `@public_api` when GA
+          class Settings < Datadog::Tracing::Contrib::Configuration::Settings
             option :enabled do |o|
               o.default { env_to_bool(Ext::ENV_ENABLED, true) }
               o.lazy

data/lib/datadog/ci/contrib/cucumber/ext.rb

@@ -4,6 +4,7 @@ module Datadog
     module Contrib
       module Cucumber
         # Cucumber integration constants
+        # TODO: mark as `@public_api` when GA, to protect from resource and tag name changes.
         module Ext
           APP = 'cucumber'.freeze
           ENV_ENABLED = 'DD_TRACE_CUCUMBER_ENABLED'.freeze