datadog 2.12.2 → 2.13.0

data/lib/datadog/appsec/contrib/devise/patches/signin_tracking_patch.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require_relative '../ext'
+require_relative '../configuration'
+require_relative '../data_extractor'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Devise
+        module Patches
+          # A patch for Devise::Authenticatable strategy with tracking functionality
+          module SigninTrackingPatch
+            def validate(resource, &block)
+              result = super
+
+              return result unless AppSec.enabled?
+              return result if @_datadog_appsec_skip_track_login_event
+              return result unless Configuration.auto_user_instrumentation_enabled?
+              return result unless AppSec.active_context
+
+              context = AppSec.active_context
+              if context.trace.nil? || context.span.nil?
+                Datadog.logger.debug { 'AppSec: unable to track signin events, due to missing trace or span' }
+                return result
+              end
+
+              context.trace.keep!
+
+              if result
+                record_successful_signin(context, resource)
+                Instrumentation.gateway.push('appsec.events.user_lifecycle', Ext::EVENT_LOGIN_SUCCESS)
+
+                return result
+              end
+
+              record_failed_signin(context, resource)
+              Instrumentation.gateway.push('appsec.events.user_lifecycle', Ext::EVENT_LOGIN_FAILURE)
+
+              result
+            end
+
+            private
+
+            def record_successful_signin(context, resource)
+              extractor = DataExtractor.new(mode: Configuration.auto_user_instrumentation_mode)
+
+              id = extractor.extract_id(resource)
+              login = extractor.extract_login(authentication_hash) || extractor.extract_login(resource)
+
+              if id
+                context.span[Ext::TAG_USR_ID] ||= id
+                context.span[Ext::TAG_DD_USR_ID] = id
+              end
+
+              context.span[Ext::TAG_LOGIN_SUCCESS_USR_LOGIN] ||= login
+              context.span[Ext::TAG_LOGIN_SUCCESS_TRACK] = 'true'
+              context.span[Ext::TAG_DD_USR_LOGIN] = login
+              context.span[Ext::TAG_DD_LOGIN_SUCCESS_MODE] = Configuration.auto_user_instrumentation_mode
+
+              # NOTE: We don't have a way to make one-shot receivers for events,
+              #       and because of that we will trigger an additional event even
+              #       if it was already done via the SDK
+              AppSec::Instrumentation.gateway.push(
+                'identity.set_user', AppSec::Instrumentation::Gateway::User.new(id, login)
+              )
+            end
+
+            def record_failed_signin(context, resource)
+              extractor = DataExtractor.new(mode: Configuration.auto_user_instrumentation_mode)
+
+              context.span[Ext::TAG_LOGIN_FAILURE_TRACK] = 'true'
+              context.span[Ext::TAG_DD_LOGIN_FAILURE_MODE] = Configuration.auto_user_instrumentation_mode
+
+              unless resource
+                login = extractor.extract_login(authentication_hash)
+
+                context.span[Ext::TAG_DD_USR_LOGIN] = login
+                context.span[Ext::TAG_LOGIN_FAILURE_USR_LOGIN] ||= login
+                context.span[Ext::TAG_LOGIN_FAILURE_USR_EXISTS] ||= 'false'
+
+                return
+              end
+
+              id = extractor.extract_id(resource)
+              login = extractor.extract_login(authentication_hash) || extractor.extract_login(resource)
+
+              if id
+                context.span[Ext::TAG_DD_USR_ID] = id
+                context.span[Ext::TAG_LOGIN_FAILURE_USR_ID] ||= id
+              end
+
+              context.span[Ext::TAG_DD_USR_LOGIN] = login
+              context.span[Ext::TAG_LOGIN_FAILURE_USR_LOGIN] ||= login
+              context.span[Ext::TAG_LOGIN_FAILURE_USR_EXISTS] ||= 'true'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/devise/patches/signup_tracking_patch.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require_relative '../ext'
+require_relative '../configuration'
+require_relative '../data_extractor'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Devise
+        module Patches
+          # A patch for Devise::RegistrationsController with tracking functionality
+          module SignupTrackingPatch
+            def create
+              return super unless AppSec.enabled?
+              return super unless Configuration.auto_user_instrumentation_enabled?
+              return super unless AppSec.active_context
+
+              super do |resource|
+                context = AppSec.active_context
+
+                if context.trace.nil? || context.span.nil?
+                  Datadog.logger.debug { 'AppSec: unable to track signup events, due to missing trace or span' }
+                  next yield(resource) if block_given?
+                end
+
+                next yield(resource) if resource.new_record? && block_given?
+
+                context.trace.keep!
+                record_successful_signup(context, resource)
+                Instrumentation.gateway.push('appsec.events.user_lifecycle', Ext::EVENT_SIGNUP)
+
+                yield(resource) if block_given?
+              end
+            end
+
+            private
+
+            def record_successful_signup(context, resource)
+              extractor = DataExtractor.new(mode: Configuration.auto_user_instrumentation_mode)
+
+              id = extractor.extract_id(resource)
+              login = extractor.extract_login(resource_params) || extractor.extract_login(resource)
+
+              context.span[Ext::TAG_SIGNUP_TRACK] = 'true'
+              context.span[Ext::TAG_DD_USR_LOGIN] = login
+              context.span[Ext::TAG_SIGNUP_USR_LOGIN] ||= login
+              context.span[Ext::TAG_DD_SIGNUP_MODE] = Configuration.auto_user_instrumentation_mode
+
+              if id
+                context.span[Ext::TAG_DD_USR_ID] = id
+
+                id_tag = resource.active_for_authentication? ? Ext::TAG_USR_ID : Ext::TAG_SIGNUP_USR_ID
+                context.span[id_tag] ||= id
+              end
+
+              # NOTE: We don't have a way to make one-shot receivers for events,
+              #       and because of that we will trigger an additional event even
+              #       if it was already done via the SDK
+              AppSec::Instrumentation.gateway.push(
+                'identity.set_user', AppSec::Instrumentation::Gateway::User.new(id, login)
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/devise/{patcher/rememberable_patch.rb → patches/skip_signin_tracking_patch.rb}

@@ -4,10 +4,10 @@ module Datadog
   module AppSec
     module Contrib
       module Devise
-        module Patcher
+        module Patches
           # To avoid tracking new sessions that are created by
           # Rememberable strategy as Login Success events.
-          module RememberablePatch
+          module SkipSigninTrackingPatch
             def validate(*args)
               @_datadog_appsec_skip_track_login_event = true
 

data/lib/datadog/appsec/contrib/devise/tracking_middleware.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require_relative 'ext'
+require_relative '../../anonymizer'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Devise
+        # A Rack middleware capable of tracking currently signed user
+        class TrackingMiddleware
+          WARDEN_KEY = 'warden'
+
+          def initialize(app)
+            @app = app
+            @devise_session_scope_keys = {}
+          end
+
+          def call(env)
+            return @app.call(env) unless AppSec.enabled?
+            return @app.call(env) unless Configuration.auto_user_instrumentation_enabled?
+            return @app.call(env) unless AppSec.active_context
+
+            unless env.key?(WARDEN_KEY)
+              Datadog.logger.debug { 'AppSec: unable to track requests, due to missing warden manager' }
+              return @app.call(env)
+            end
+
+            context = AppSec.active_context
+            if context.trace.nil? || context.span.nil?
+              Datadog.logger.debug { 'AppSec: unable to track requests, due to missing trace or span' }
+              return @app.call(env)
+            end
+
+            id = transform(extract_id(env[WARDEN_KEY]))
+            if id
+              unless context.span.has_tag?(Ext::TAG_USR_ID)
+                context.span[Ext::TAG_USR_ID] = id
+                AppSec::Instrumentation.gateway.push(
+                  'identity.set_user', AppSec::Instrumentation::Gateway::User.new(id, nil)
+                )
+              end
+
+              context.span[Ext::TAG_DD_USR_ID] = id.to_s
+              context.span[Ext::TAG_DD_COLLECTION_MODE] ||= Configuration.auto_user_instrumentation_mode
+            end
+
+            @app.call(env)
+          end
+
+          private
+
+          def extract_id(warden)
+            session_serializer = warden.session_serializer
+
+            key = session_key_for(session_serializer, ::Devise.default_scope)
+            id = session_serializer.session[key]&.dig(0, 0)
+
+            return id if ::Devise.mappings.size == 1
+            return "#{::Devise.default_scope}:#{id}" if id
+
+            ::Devise.mappings.each_key do |scope|
+              next if scope == ::Devise.default_scope
+
+              key = session_key_for(session_serializer, scope)
+              id = session_serializer.session[key]&.dig(0, 0)
+
+              return "#{scope}:#{id}" if id
+            end
+
+            nil
+          end
+
+          def session_key_for(session_serializer, scope)
+            @devise_session_scope_keys[scope] ||= session_serializer.key_for(scope)
+          end
+
+          def transform(value)
+            return if value.nil?
+            return value.to_s unless anonymize?
+
+            Anonymizer.anonimyze(value.to_s)
+          end
+
+          def anonymize?
+            Configuration.auto_user_instrumentation_mode ==
+              AppSec::Configuration::Settings::ANONYMIZATION_AUTO_USER_INSTRUMENTATION_MODE
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rack/ext.rb

@@ -6,6 +6,20 @@ module Datadog
       module Rack
         # Rack integration constants
         module Ext
+          COLLECTABLE_REQUEST_HEADERS = [
+            'accept',
+            'akamai-user-risk',
+            'cf-ray',
+            'cloudfront-viewer-ja3-fingerprint',
+            'content-type',
+            'user-agent',
+            'x-amzn-trace-Id',
+            'x-appgw-trace-id',
+            'x-cloud-trace-context',
+            'x-sigsci-requestid',
+            'x-sigsci-tags'
+          ].freeze
+
           IDENTITY_COLLECTABLE_REQUEST_HEADERS = [
             'accept-encoding',
             'accept-language',

data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb

@@ -18,6 +18,7 @@ module Datadog
                 watch_request(gateway)
                 watch_response(gateway)
                 watch_request_body(gateway)
+                watch_request_finish(gateway)
               end
 
               def watch_request(gateway = Instrumentation.gateway)
@@ -119,12 +120,18 @@ module Datadog
               def watch_request_finish(gateway = Instrumentation.gateway)
                 gateway.watch('rack.request.finish', :appsec) do |stack, gateway_request|
                   context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
-                  next stack.call(gateway_request.request) if context.span.nil? || !gateway.pushed?('identity.set_user')
+
+                  if context.span.nil? || !gateway.pushed?('appsec.events.user_lifecycle')
+                    next stack.call(gateway_request.request)
+                  end
 
                   gateway_request.headers.each do |name, value|
-                    next unless Ext::IDENTITY_COLLECTABLE_REQUEST_HEADERS.include?(name)
+                    if !Ext::COLLECTABLE_REQUEST_HEADERS.include?(name) &&
+                        !Ext::IDENTITY_COLLECTABLE_REQUEST_HEADERS.include?(name)
+                      next
+                    end
 
-                    context.span["http.request.headers.#{name}"] = value
+                    context.span["http.request.headers.#{name}"] ||= value
                   end
 
                   stack.call(gateway_request.request)

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -150,8 +150,6 @@ module Datadog
             return unless trace && span
 
             span.set_metric(Datadog::AppSec::Ext::TAG_APPSEC_ENABLED, 1)
-            # We add this tag when ASM standalone is enabled to make sure we don't bill APM
-            span.set_metric(Datadog::AppSec::Ext::TAG_APM_ENABLED, 0) if Datadog.configuration.appsec.standalone.enabled
             span.set_tag('_dd.runtime_family', 'ruby')
             span.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
 

data/lib/datadog/appsec/event.rb

@@ -187,7 +187,7 @@ module Datadog
             Datadog::Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
             Datadog::Tracing::Sampling::Ext::Decision::ASM
           )
-          trace.set_tag(Datadog::AppSec::Ext::TAG_DISTRIBUTED_APPSEC_EVENT, '1')
+          trace.set_distributed_source(Datadog::AppSec::Ext::PRODUCT_BIT)
         end
       end
     end

data/lib/datadog/appsec/ext.rb

@@ -7,13 +7,15 @@ module Datadog
       RASP_LFI = 'lfi'
       RASP_SSRF = 'ssrf'
 
+      PRODUCT_BIT = 0b00000010
+
       INTERRUPT = :datadog_appsec_interrupt
       CONTEXT_KEY = 'datadog.appsec.context'
       ACTIVE_CONTEXT_KEY = :datadog_appsec_active_context
+      EXPLOIT_PREVENTION_EVENT_CATEGORY = 'exploit'
 
       TAG_APPSEC_ENABLED = '_dd.appsec.enabled'
-      TAG_APM_ENABLED = '_dd.apm.enabled'
-      TAG_DISTRIBUTED_APPSEC_EVENT = '_dd.p.appsec'
+      TAG_METASTRUCT_STACK_TRACE = '_dd.stack'
 
       TELEMETRY_METRICS_NAMESPACE = 'appsec'
     end

data/lib/datadog/appsec/instrumentation/gateway/argument.rb

@@ -9,11 +9,13 @@ module Datadog
 
         # Gateway User argument
         class User < Argument
-          attr_reader :id
+          attr_reader :id, :login
 
-          def initialize(id)
+          def initialize(id, login)
             super()
+
             @id = id
+            @login = login
           end
         end
       end

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -19,9 +19,14 @@ module Datadog
               gateway.watch('identity.set_user', :appsec) do |stack, user|
                 context = Datadog::AppSec.active_context
 
-                persistent_data = {
-                  'usr.id' => user.id
-                }
+                if user.id.nil? && user.login.nil?
+                  Datadog.logger.debug { 'AppSec: skipping WAF check because no user information was provided' }
+                  next stack.call(user)
+                end
+
+                persistent_data = {}
+                persistent_data['usr.id'] = user.id if user.id
+                persistent_data['usr.login'] = user.login if user.login
 
                 result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 

data/lib/datadog/appsec/security_engine/runner.rb

@@ -24,13 +24,13 @@ module Datadog
           persistent_data.reject! do |_, v|
             next false if v.is_a?(TrueClass) || v.is_a?(FalseClass)
 
-            v.nil? ? true : v.empty?
+            v.nil? || v.empty?
           end
 
           ephemeral_data.reject! do |_, v|
             next false if v.is_a?(TrueClass) || v.is_a?(FalseClass)
 
-            v.nil? ? true : v.empty?
+            v.nil? || v.empty?
           end
 
           _code, result = try_run(persistent_data, ephemeral_data, timeout)

data/lib/datadog/appsec/utils.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require_relative 'utils/trace_operation'
-
 module Datadog
   module AppSec
     # Utilities for AppSec

data/lib/datadog/core/configuration/components.rb

@@ -45,6 +45,7 @@ module Datadog
             options = { enabled: settings.runtime_metrics.enabled }
             options[:statsd] = settings.runtime_metrics.statsd unless settings.runtime_metrics.statsd.nil?
             options[:services] = [settings.service] unless settings.service.nil?
+            options[:experimental_runtime_id_enabled] = settings.runtime_metrics.experimental_runtime_id_enabled
 
             Core::Runtime::Metrics.new(logger: logger, **options)
           end
@@ -101,7 +102,7 @@ module Datadog
           agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
 
           # Exposes agent capability information for detection by any components
-          @agent_info = Core::Environment::AgentInfo.new(agent_settings)
+          @agent_info = Core::Environment::AgentInfo.new(agent_settings, logger: @logger)
 
           @telemetry = self.class.build_telemetry(settings, agent_settings, @logger)
 

data/lib/datadog/core/configuration/ext.rb

@@ -18,6 +18,10 @@ module Datadog
           ENV_DEFAULT_PORT = 'DD_METRIC_AGENT_PORT'
         end
 
+        module APM
+          ENV_TRACING_ENABLED = 'DD_APM_TRACING_ENABLED'
+        end
+
         module Agent
           ENV_DEFAULT_HOST = 'DD_AGENT_HOST'
           # Some env vars have "trace" in them, but they apply to all products

data/lib/datadog/core/configuration/options.rb

@@ -45,7 +45,7 @@ module Datadog
               option_name.to_sym => proc do
                 get_option(option_name)
               end,
-              "#{option_name}=".to_sym => proc do |value|
+              :"#{option_name}=" => proc do |value|
                 set_option(option_name, value)
               end
             }
@@ -117,7 +117,7 @@ module Datadog
           end
 
           def resolved_env(name)
-            return options[name].resolved_env if options.key?(name)
+            options[name].resolved_env if options.key?(name)
           end
 
           def assert_valid_option!(name)

data/lib/datadog/core/configuration/settings.rb

@@ -570,6 +570,12 @@ module Datadog
             o.type :bool
           end
 
+          option :experimental_runtime_id_enabled do |o|
+            o.type :bool
+            o.env 'DD_TRACE_EXPERIMENTAL_RUNTIME_ID_ENABLED'
+            o.default false
+          end
+
           option :opts, default: {}, type: :hash
           option :statsd
         end
@@ -619,38 +625,31 @@ module Datadog
           o.type :hash, nilable: true
           o.env [Core::Environment::Ext::ENV_TAGS, Core::Environment::Ext::ENV_OTEL_RESOURCE_ATTRIBUTES]
           o.env_parser do |env_value|
-            values = if env_value.include?(',')
-                       env_value.split(',')
-                     else
-                       env_value.split(' ') # rubocop:disable Style/RedundantArgument
-                     end
-            values.map! do |v|
-              v.gsub!(/\A[\s,]*|[\s,]*\Z/, '')
-
-              v.empty? ? nil : v
-            end
-
-            values.compact!
-            values.each_with_object({}) do |tag, tags|
-              key, value = tag.split(':', 2)
-              if value.nil?
-                # support tags/attributes delimited by the OpenTelemetry separator (`=`)
-                key, value = tag.split('=', 2)
-              end
-              next if value.nil? || value.empty?
-
-              # maps OpenTelemetry semantic attributes to Datadog tags
-              case key.downcase
-              when 'deployment.environment'
-                tags['env'] = value
-              when 'service.version'
-                tags['version'] = value
-              when 'service.name'
-                tags['service'] = value
-              else
-                tags[key] = value
+            # Parses a string containing key-value pairs and returns a hash.
+            # Key-value pairs are delimited by ':' OR `=`, and pairs are separated by whitespace, comma, OR BOTH.
+            result = {}
+            unless env_value.nil? || env_value.empty?
+              # falling back to comma as separator
+              sep = env_value.include?(',') ? ',' : ' '
+              # split by separator
+              env_value.split(sep).each do |tag|
+                tag.strip!
+                next if tag.empty?
+
+                # tag by : or = (for OpenTelemetry)
+                key, val = tag.split(/[:=]/, 2).map(&:strip)
+                val ||= ''
+                # maps OpenTelemetry semantic attributes to Datadog tags
+                key = case key.downcase
+                      when 'deployment.environment' then 'env'
+                      when 'service.version' then 'version'
+                      when 'service.name' then 'service'
+                      else key
+                      end
+                result[key] = val unless key.empty?
               end
             end
+            result
           end
           o.setter do |new_value, old_value|
             raw_tags = new_value || {}
@@ -958,6 +957,30 @@ module Datadog
           end
         end
 
+        # Tracer specific configuration starting with APM (e.g. DD_APM_TRACING_ENABLED).
+        # @public_api
+        settings :apm do
+          # Tracing as a transport
+          # @public_api
+          settings :tracing do
+            # Enables tracing as transport.
+            # Disabling it will set sampling priority to -1 (FORCE_DROP) on most traces,
+            # (which tells to the agent to drop these traces)
+            # except heartbeat ones (1 per minute) and manually kept ones (sampling priority to 2) (e.g. appsec events)
+            #
+            # This is different than `DD_TRACE_ENABLED`, which completely disables tracing (sends no trace at all),
+            # while this will send heartbeat traces (1 per minute) so that the service is considered alive in the backend.
+            #
+            # @default `DD_APM_TRACING_ENABLED` environment variable, otherwise `true`
+            # @return [Boolean]
+            option :enabled do |o|
+              o.env Configuration::Ext::APM::ENV_TRACING_ENABLED
+              o.default true
+              o.type :bool
+            end
+          end
+        end
+
         # TODO: Tracing should manage its own settings.
         #       Keep this extension here for now to keep things working.
         extend Datadog::Tracing::Configuration::Settings

data/lib/datadog/core/environment/agent_info.rb

@@ -51,11 +51,12 @@ module Datadog
       #
       # @see https://github.com/DataDog/datadog-agent/blob/f07df0a3c1fca0c83b5a15f553bd994091b0c8ac/pkg/trace/api/info.go#L20
       class AgentInfo
-        attr_reader :agent_settings
+        attr_reader :agent_settings, :logger
 
-        def initialize(agent_settings)
+        def initialize(agent_settings, logger:)
           @agent_settings = agent_settings
-          @client = Remote::Transport::HTTP.root(agent_settings: agent_settings)
+          @logger = logger
+          @client = Remote::Transport::HTTP.root(agent_settings: agent_settings, logger: logger)
         end
 
         # Fetches the information from the agent.

data/lib/datadog/core/remote/component.rb

@@ -18,11 +18,8 @@ module Datadog
         def initialize(settings, capabilities, agent_settings, logger:)
           @logger = logger
 
-          transport_options = {}
-          transport_options[:agent_settings] = agent_settings if agent_settings
-
-          negotiation = Negotiation.new(settings, agent_settings)
-          transport_v7 = Datadog::Core::Remote::Transport::HTTP.v7(**transport_options) # steep:ignore
+          negotiation = Negotiation.new(settings, agent_settings, logger: logger)
+          transport_v7 = Datadog::Core::Remote::Transport::HTTP.v7(agent_settings: agent_settings, logger: logger)
 
           @barrier = Barrier.new(settings.remote.boot_timeout_seconds)
 
@@ -49,7 +46,7 @@ module Datadog
               # In case of unexpected errors, reset the negotiation object
               # given external conditions have changed and the negotiation
               # negotiation object stores error logging state that should be reset.
-              negotiation = Negotiation.new(settings, agent_settings)
+              negotiation = Negotiation.new(settings, agent_settings, logger: logger)
 
               # Transient errors due to network or agent. Logged the error but not via telemetry
               logger.error do

data/lib/datadog/core/remote/configuration/repository.rb

@@ -272,7 +272,8 @@ module Datadog
 
               return deleted(path, previous) if previous && content.nil?
               return inserted(path, content) if content && previous.nil?
-              return updated(path, content, previous) if content && previous
+
+              updated(path, content, previous) if content && previous
             end
 
             def deleted(path, previous)