datadog 2.7.1 → 2.17.0

data/lib/datadog/appsec/security_event.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # A class that represents a security event of any kind. It could be an event
+    # representing an attack or fingerprinting results as derivatives or an API
+    # security check with extracted schema.
+    class SecurityEvent
+      SCHEMA_KEY_PREFIX = '_dd.appsec.s.'
+      FINGERPRINT_KEY_PREFIX = '_dd.appsec.fp.'
+
+      attr_reader :waf_result, :trace, :span
+
+      def initialize(waf_result, trace:, span:)
+        @waf_result = waf_result
+        @trace = trace
+        @span = span
+      end
+
+      def attack?
+        return @is_attack if defined?(@is_attack)
+
+        @is_attack = @waf_result.is_a?(SecurityEngine::Result::Match)
+      end
+
+      def schema?
+        return @has_schema if defined?(@has_schema)
+
+        @has_schema = @waf_result.derivatives.any? { |name, _| name.start_with?(SCHEMA_KEY_PREFIX) }
+      end
+
+      def fingerprint?
+        return @has_fingerprint if defined?(@has_fingerprint)
+
+        @has_fingerprint = @waf_result.derivatives.any? { |name, _| name.start_with?(FINGERPRINT_KEY_PREFIX) }
+      end
+    end
+  end
+end

data/lib/datadog/appsec/utils.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require_relative 'utils/trace_operation'
-
 module Datadog
   module AppSec
     # Utilities for AppSec

data/lib/datadog/appsec.rb

@@ -2,7 +2,7 @@
 
 require_relative 'appsec/configuration'
 require_relative 'appsec/extensions'
-require_relative 'appsec/scope'
+require_relative 'appsec/context'
 require_relative 'appsec/ext'
 require_relative 'appsec/utils'
 
@@ -14,32 +14,41 @@ module Datadog
         Datadog.configuration.appsec.enabled
       end
 
-      def active_scope
-        Datadog::AppSec::Scope.active_scope
+      def rasp_enabled?
+        Datadog.configuration.appsec.rasp_enabled
       end
 
-      def processor
-        appsec_component = components.appsec
+      def active_context
+        Datadog::AppSec::Context.active
+      end
 
-        appsec_component.processor if appsec_component
+      def telemetry
+        components.appsec&.telemetry
       end
 
-      def reconfigure(ruleset:, actions:, telemetry:)
-        appsec_component = components.appsec
+      def processor
+        components.appsec&.processor
+      end
 
+      def reconfigure(ruleset:, telemetry:)
+        appsec_component = components.appsec
         return unless appsec_component
 
-        appsec_component.reconfigure(ruleset: ruleset, actions: actions, telemetry: telemetry)
+        appsec_component.reconfigure(ruleset: ruleset, telemetry: telemetry)
       end
 
       def reconfigure_lock(&block)
         appsec_component = components.appsec
-
         return unless appsec_component
 
         appsec_component.reconfigure_lock(&block)
       end
 
+      def perform_api_security_check?
+        Datadog.configuration.appsec.api_security.enabled &&
+          Datadog.configuration.appsec.api_security.sample_rate.sample?
+      end
+
       private
 
       def components
@@ -56,7 +65,11 @@ end
 require_relative 'appsec/contrib/rack/integration'
 require_relative 'appsec/contrib/sinatra/integration'
 require_relative 'appsec/contrib/rails/integration'
+require_relative 'appsec/contrib/active_record/integration'
 require_relative 'appsec/contrib/devise/integration'
 require_relative 'appsec/contrib/graphql/integration'
+require_relative 'appsec/contrib/faraday/integration'
+require_relative 'appsec/contrib/excon/integration'
+require_relative 'appsec/contrib/rest_client/integration'
 
 require_relative 'appsec/autoload'

data/lib/datadog/auto_instrument.rb

@@ -6,6 +6,9 @@
 require_relative '../datadog'
 require_relative 'tracing/contrib/auto_instrument'
 
+# DI is not loaded on Ruby 2.5 and JRuby
+Datadog::DI::Contrib.load_now_or_later if defined?(Datadog::DI::Contrib)
+
 Datadog::Profiling.start_if_enabled
 
 module Datadog

data/lib/datadog/core/buffer/random.rb

@@ -40,7 +40,23 @@ module Datadog
           add_all!(underflow) unless underflow.nil?
 
           # Iteratively replace items, to ensure pseudo-random replacement.
-          overflow.each { |item| replace!(item) } unless overflow.nil?
+          overflow&.each { |item| replace!(item) }
+        end
+
+        def unshift(*items)
+          # TODO The existing concat implementation does not always append
+          # to the end of the buffer - if the buffer is full, a random
+          # item is deleted and the new item is added in the position of
+          # removed item.
+          # Therefore, if we want to preserve the item order, concat
+          # would also need to be changed to maintain order.
+          # With the existing implementation, the idea is to not move
+          # existing items around, which is what sets unshift apart from
+          # concat to begin with.
+          #
+          # Since this method currently delegates to +concat+, it does not
+          # have a matching definition in the thread-safe worker.
+          concat(items)
         end
 
         # Stored items are returned and the local buffer is reset.
@@ -78,7 +94,7 @@ module Datadog
           underflow = nil
           overflow = nil
 
-          overflow_size = @max_size > 0 ? (@items.length + items.length) - @max_size : 0
+          overflow_size = (@max_size > 0) ? (@items.length + items.length) - @max_size : 0
 
           if overflow_size > 0
             # Items will overflow

data/lib/datadog/core/configuration/agent_settings_resolver.rb

@@ -19,21 +19,49 @@ module Datadog
       # Whenever there is a conflict (different configurations are provided in different orders), it MUST warn the users
       # about it and pick a value based on the following priority: code > environment variable > defaults.
       class AgentSettingsResolver
-        AgentSettings = Struct.new(
-          :adapter,
-          :ssl,
-          :hostname,
-          :port,
-          :uds_path,
-          :timeout_seconds,
-          keyword_init: true
-        ) do
-          def initialize(*)
-            super
+        # Immutable container for the resulting settings
+        class AgentSettings
+          attr_reader :adapter, :ssl, :hostname, :port, :uds_path, :timeout_seconds
+
+          def initialize(adapter: nil, ssl: nil, hostname: nil, port: nil, uds_path: nil, timeout_seconds: nil)
+            @adapter = adapter
+            @ssl = ssl
+            @hostname = hostname
+            @port = port
+            @uds_path = uds_path
+            @timeout_seconds = timeout_seconds
             freeze
           end
+
+          def url
+            case adapter
+            when Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
+              hostname = self.hostname
+              hostname = "[#{hostname}]" if IPV6_REGEXP.match?(hostname)
+              "#{ssl ? "https" : "http"}://#{hostname}:#{port}/"
+            when Datadog::Core::Configuration::Ext::Agent::UnixSocket::ADAPTER
+              "unix://#{uds_path}"
+            else
+              raise ArgumentError, "Unexpected adapter: #{adapter}"
+            end
+          end
+
+          def ==(other)
+            self.class == other.class &&
+              adapter == other.adapter &&
+              ssl == other.ssl &&
+              hostname == other.hostname &&
+              port == other.port &&
+              uds_path == other.uds_path &&
+              timeout_seconds == other.timeout_seconds
+          end
         end
 
+        # IPv6 regular expression from
+        # https://stackoverflow.com/questions/53497/regular-expression-that-matches-valid-ipv6-addresses
+        # Does not match IPv4 addresses.
+        IPV6_REGEXP = /\A(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\z)/.freeze # rubocop:disable Layout/LineLength
+
         def self.call(settings, logger: Datadog.logger)
           new(settings, logger: logger).send(:call)
         end
@@ -130,7 +158,7 @@ module Datadog
               value: settings.agent.timeout_seconds,
             ),
             try_parsing_as_integer(
-              friendly_name: "#{Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_TIMEOUT_SECONDS} "\
+              friendly_name: "#{Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_TIMEOUT_SECONDS} " \
                 'environment variable',
               value: ENV[Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_TIMEOUT_SECONDS],
             )
@@ -310,13 +338,13 @@ module Datadog
           log_warning(
             'Configuration mismatch: values differ between ' \
             "#{detected_configurations_in_priority_order
-              .map { |config| "#{config.friendly_name} (#{config.value.inspect})" }.join(' and ')}" \
+              .map { |config| "#{config.friendly_name} (#{config.value.inspect})" }.join(" and ")}" \
             ". Using #{detected_configurations_in_priority_order.first.value.inspect} and ignoring other configuration."
           )
         end
 
         def log_warning(message)
-          logger.warn(message) if logger
+          logger&.warn(message)
         end
 
         def http_scheme?(uri)

data/lib/datadog/core/configuration/agentless_settings_resolver.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+# rubocop:disable Style/*
+
+require 'uri'
+
+require_relative 'agent_settings_resolver'
+
+module Datadog
+  module Core
+    module Configuration
+      # Agent settings resolver for agentless operations (currently, telemetry
+      # in agentless mode).
+      #
+      # The terminology gets a little confusing here, but transports communicate
+      # with servers which are - for most components in the tracer - the
+      # (local) agent. Hence, "agent settings" to refer to where the server
+      # is located. Telemetry supports sending to the local agent but also
+      # implements agentless mode where it sends directly to Datadog intake
+      # endpoints. The agentless mode is configured using different settings,
+      # and this class produces AgentSettings instances when in agentless mode.
+      #
+      # Agentless settings resolver uses the following configuration sources:
+      #
+      # 1. url_override constructor parameter, if provided
+      # 2. Built-in default host/port/TLS settings for the backend
+      #    intake endpoint
+      #
+      # The agentless resolver does NOT use agent settings (since it is
+      # for agentless operation), specifically it ignores:
+      #
+      # - c.agent.host
+      # - DD_AGENT_HOST
+      # - c.agent.port
+      # - DD_AGENT_PORT
+      #
+      # However, agentless resolver does respect the timeout specified via
+      # c.agent.timeout_seconds or DD_TRACE_AGENT_TIMEOUT_SECONDS.
+      class AgentlessSettingsResolver < AgentSettingsResolver
+        # To avoid coupling this class to telemetry, the URL override is
+        # taken here as a parameter instead of being read out of
+        # c.telemetry.agentless_url_override. For the same reason, the
+        # +url_override_source+ parameter should be set to the string
+        # "c.telemetry.agentless_url_override".
+        def self.call(settings, host_prefix:, url_override: nil, url_override_source: nil, logger: Datadog.logger)
+          new(
+            settings,
+            host_prefix: host_prefix,
+            url_override: url_override,
+            url_override_source: url_override_source,
+            logger: logger
+          ).send(:call)
+        end
+
+        private
+
+        attr_reader \
+          :host_prefix,
+          :url_override,
+          :url_override_source
+
+        def initialize(settings, host_prefix:, url_override: nil, url_override_source: nil, logger: Datadog.logger)
+          if url_override && url_override_source.nil?
+            raise ArgumentError, 'url_override_source must be provided when url_override is provided'
+          end
+
+          super(settings, logger: logger)
+
+          @host_prefix = host_prefix
+          @url_override = url_override
+          @url_override_source = url_override_source
+        end
+
+        def hostname
+          if should_use_uds?
+            nil
+          else
+            configured_hostname || "#{host_prefix}.#{settings.site}"
+          end
+        end
+
+        def configured_hostname
+          return @configured_hostname if defined?(@configured_hostname)
+
+          if should_use_uds?
+            nil
+          else
+            @configured_hostname = (parsed_url.hostname if parsed_url)
+          end
+        end
+
+        def configured_port
+          return @configured_port if defined?(@configured_port)
+
+          @configured_port = (parsed_url.port if parsed_url)
+        end
+
+        # Note that this method should always return true or false
+        def ssl?
+          if configured_hostname
+            configured_ssl || false
+          else
+            if should_use_uds?
+              false
+            else
+              # If no hostname is specified, we are communicating with the
+              # default Datadog intake, which uses TLS.
+              true
+            end
+          end
+        end
+
+        # Note that this method can return nil
+        def configured_ssl
+          return @configured_ssl if defined?(@configured_ssl)
+
+          @configured_ssl = (parsed_url_ssl? if parsed_url)
+        end
+
+        def port
+          if configured_port
+            configured_port
+          else
+            if should_use_uds?
+              nil
+            else
+              # If no hostname is specified, we are communicating with the
+              # default Datadog intake, which exists on port 443.
+              443
+            end
+          end
+        end
+
+        def mixed_http_and_uds
+          false
+        end
+
+        def configured_uds_path
+          return @configured_uds_path if defined?(@configured_uds_path)
+
+          parsed_url_uds_path
+        end
+
+        def can_use_uds?
+          # While in theory agentless transport could communicate via UDS,
+          # in practice "agentless" means we are communicating with Datadog
+          # infrastructure which is always remote.
+          # Permit UDS for proxy usage?
+          !configured_uds_path.nil?
+        end
+
+        def parsed_url
+          return @parsed_url if defined?(@parsed_url)
+
+          @parsed_url =
+            if @url_override
+              parsed = URI.parse(@url_override)
+
+              # Agentless URL should never refer to a UDS?
+              if http_scheme?(parsed) || unix_scheme?(parsed)
+                parsed
+              else
+                log_warning(
+                  "Invalid URI scheme '#{parsed.scheme}' for #{url_override_source}. " \
+                  "Ignoring the contents of #{url_override_source}."
+                )
+                nil
+              end
+            end
+        end
+      end
+    end
+  end
+end
+
+# rubocop:enable Style/*

data/lib/datadog/core/configuration/components.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'agent_settings_resolver'
+require_relative 'components_state'
 require_relative 'ext'
 require_relative '../diagnostics/environment_logger'
 require_relative '../diagnostics/health'
@@ -8,12 +9,15 @@ require_relative '../logger'
 require_relative '../runtime/metrics'
 require_relative '../telemetry/component'
 require_relative '../workers/runtime_metrics'
-
 require_relative '../remote/component'
 require_relative '../../tracing/component'
 require_relative '../../profiling/component'
 require_relative '../../appsec/component'
+require_relative '../../di/component'
+require_relative '../../error_tracking/component'
 require_relative '../crashtracking/component'
+require_relative '../environment/agent_info'
+require_relative '../process_discovery'
 
 module Datadog
   module Core
@@ -23,12 +27,12 @@ module Datadog
         class << self
           include Datadog::Tracing::Component
 
-          def build_health_metrics(settings)
+          def build_health_metrics(settings, logger, telemetry)
             settings = settings.health_metrics
-            options = { enabled: settings.enabled }
+            options = {enabled: settings.enabled}
             options[:statsd] = settings.statsd unless settings.statsd.nil?
 
-            Core::Diagnostics::Health::Metrics.new(**options)
+            Core::Diagnostics::Health::Metrics.new(telemetry: telemetry, logger: logger, **options)
           end
 
           def build_logger(settings)
@@ -38,22 +42,24 @@ module Datadog
             logger
           end
 
-          def build_runtime_metrics(settings)
-            options = { enabled: settings.runtime_metrics.enabled }
+          def build_runtime_metrics(settings, logger, telemetry)
+            options = {enabled: settings.runtime_metrics.enabled}
             options[:statsd] = settings.runtime_metrics.statsd unless settings.runtime_metrics.statsd.nil?
             options[:services] = [settings.service] unless settings.service.nil?
+            options[:experimental_runtime_id_enabled] = settings.runtime_metrics.experimental_runtime_id_enabled
 
-            Core::Runtime::Metrics.new(**options)
+            Core::Runtime::Metrics.new(logger: logger, telemetry: telemetry, **options)
           end
 
-          def build_runtime_metrics_worker(settings)
+          def build_runtime_metrics_worker(settings, logger, telemetry)
             # NOTE: Should we just ignore building the worker if its not enabled?
             options = settings.runtime_metrics.opts.merge(
               enabled: settings.runtime_metrics.enabled,
-              metrics: build_runtime_metrics(settings)
+              metrics: build_runtime_metrics(settings, logger, telemetry),
+              logger: logger,
             )
 
-            Core::Workers::RuntimeMetrics.new(options)
+            Core::Workers::RuntimeMetrics.new(telemetry: telemetry, **options)
           end
 
           def build_telemetry(settings, agent_settings, logger)
@@ -63,7 +69,7 @@ module Datadog
           def build_crashtracker(settings, agent_settings, logger:)
             return unless settings.crashtracking.enabled
 
-            if (libdatadog_api_failure = Datadog::Core::Crashtracking::Component::LIBDATADOG_API_FAILURE)
+            if (libdatadog_api_failure = Datadog::Core::LIBDATADOG_API_FAILURE)
               logger.debug("Cannot enable crashtracking: #{libdatadog_api_failure}")
               return
             end
@@ -83,7 +89,10 @@ module Datadog
           :telemetry,
           :tracer,
           :crashtracker,
-          :appsec
+          :error_tracking,
+          :dynamic_instrumentation,
+          :appsec,
+          :agent_info
 
         def initialize(settings)
           @logger = self.class.build_logger(settings)
@@ -94,28 +103,39 @@ module Datadog
           # the Core resolver from within your product/component's namespace.
           agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
 
+          # Exposes agent capability information for detection by any components
+          @agent_info = Core::Environment::AgentInfo.new(agent_settings, logger: @logger)
+
           @telemetry = self.class.build_telemetry(settings, agent_settings, @logger)
 
-          @remote = Remote::Component.build(settings, agent_settings, telemetry: telemetry)
+          @remote = Remote::Component.build(settings, agent_settings, logger: @logger, telemetry: telemetry)
           @tracer = self.class.build_tracer(settings, agent_settings, logger: @logger)
           @crashtracker = self.class.build_crashtracker(settings, agent_settings, logger: @logger)
 
           @profiler, profiler_logger_extra = Datadog::Profiling::Component.build_profiler_component(
             settings: settings,
             agent_settings: agent_settings,
-            optional_tracer: @tracer
+            optional_tracer: @tracer,
+            logger: @logger,
           )
           @environment_logger_extra.merge!(profiler_logger_extra) if profiler_logger_extra
 
-          @runtime_metrics = self.class.build_runtime_metrics_worker(settings)
-          @health_metrics = self.class.build_health_metrics(settings)
+          @runtime_metrics = self.class.build_runtime_metrics_worker(settings, @logger, telemetry)
+          @health_metrics = self.class.build_health_metrics(settings, @logger, telemetry)
           @appsec = Datadog::AppSec::Component.build_appsec_component(settings, telemetry: telemetry)
+          @dynamic_instrumentation = Datadog::DI::Component.build(settings, agent_settings, @logger, telemetry: telemetry)
+          @error_tracking = Datadog::ErrorTracking::Component.build(settings, @tracer, @logger)
+          @environment_logger_extra[:dynamic_instrumentation_enabled] = !!@dynamic_instrumentation
+          # TODO: Re-enable this once we have updated libdatadog to 17.1
+          # @process_discovery_fd = Core::ProcessDiscovery.get_and_store_metadata(settings, @logger)
 
           self.class.configure_tracing(settings)
         end
 
         # Starts up components
-        def startup!(settings)
+        def startup!(settings, old_state: nil)
+          telemetry.start(old_state&.telemetry_enabled?)
+
           if settings.profiling.enabled
             if profiler
               profiler.start
@@ -126,6 +146,16 @@ module Datadog
             end
           end
 
+          if settings.remote.enabled && old_state&.remote_started?
+            # The library was reconfigured and previously it already started
+            # the remote component (i.e., it received at least one request
+            # through the installed Rack middleware which started the remote).
+            # If the new configuration also has remote enabled, start the
+            # new remote right away.
+            # remote should always be not nil here but steep doesn't know this.
+            remote&.start
+          end
+
           Core::Diagnostics::EnvironmentLogger.collect_and_log!(@environment_logger_extra)
         end
 
@@ -134,17 +164,20 @@ module Datadog
         # and avoid tearing down parts still in use.
         def shutdown!(replacement = nil)
           # Shutdown remote configuration
-          remote.shutdown! if remote
+          remote&.shutdown!
+
+          # Shutdown DI after remote, since remote config triggers DI operations.
+          dynamic_instrumentation&.shutdown!
 
           # Decommission AppSec
-          appsec.shutdown! if appsec
+          appsec&.shutdown!
 
           # Shutdown the old tracer, unless it's still being used.
           # (e.g. a custom tracer instance passed in.)
-          tracer.shutdown! unless replacement && tracer == replacement.tracer
+          tracer.shutdown! unless replacement && tracer.equal?(replacement.tracer)
 
           # Shutdown old profiler
-          profiler.shutdown! unless profiler.nil?
+          profiler&.shutdown!
 
           # Shutdown workers
           runtime_metrics.stop(true, close_metrics: false)
@@ -162,21 +195,32 @@ module Datadog
             health_metrics.statsd
           ].compact.uniq
 
-          new_statsd =  if replacement
-                          [
-                            replacement.runtime_metrics.metrics.statsd,
-                            replacement.health_metrics.statsd
-                          ].compact.uniq
-                        else
-                          []
-                        end
+          new_statsd = if replacement
+            [
+              replacement.runtime_metrics.metrics.statsd,
+              replacement.health_metrics.statsd
+            ].compact.uniq
+          else
+            []
+          end
 
           unused_statsd = (old_statsd - (old_statsd & new_statsd))
           unused_statsd.each(&:close)
 
-          # enqueue closing event before stopping telemetry so it will be send out on shutdown
-          telemetry.emit_closing! unless replacement
-          telemetry.stop!
+          # enqueue closing event before stopping telemetry so it will be sent out on shutdown
+          telemetry.emit_closing! unless replacement&.telemetry&.enabled
+          telemetry.shutdown!
+
+          # TODO: Re-enable this once we have updated libdatadog to 17.1
+          # Core::ProcessDiscovery._native_close_tracer_memfd(@process_discovery_fd, @logger) if @process_discovery_fd
+        end
+
+        # Returns the current state of various components.
+        def state
+          ComponentsState.new(
+            telemetry_enabled: telemetry.enabled,
+            remote_started: remote&.started?,
+          )
         end
       end
     end

data/lib/datadog/core/configuration/components_state.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Configuration
+      # Stores the state of component tree when replacing the tree.
+      class ComponentsState
+        def initialize(telemetry_enabled:, remote_started:)
+          @telemetry_enabled = !!telemetry_enabled
+          @remote_started = !!remote_started
+        end
+
+        def telemetry_enabled?
+          @telemetry_enabled
+        end
+
+        def remote_started?
+          @remote_started
+        end
+      end
+    end
+  end
+end