RubyGems - datadog - Versions diffs - 2.22.0 → 2.23.0 - Mend

datadog 2.22.0 → 2.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (152) hide show

checksums.yaml +4 -4
data/CHANGELOG.md +59 -2
data/ext/LIBDATADOG_DEVELOPMENT.md +1 -58
data/ext/datadog_profiling_native_extension/collectors_stack.c +4 -0
data/ext/datadog_profiling_native_extension/datadog_ruby_common.h +1 -1
data/ext/datadog_profiling_native_extension/extconf.rb +6 -4
data/ext/datadog_profiling_native_extension/heap_recorder.c +1 -1
data/ext/libdatadog_api/datadog_ruby_common.h +1 -1
data/ext/libdatadog_api/feature_flags.c +554 -0
data/ext/libdatadog_api/feature_flags.h +5 -0
data/ext/libdatadog_api/init.c +2 -0
data/ext/libdatadog_api/library_config.c +12 -11
data/ext/libdatadog_extconf_helpers.rb +1 -1
data/lib/datadog/appsec/api_security/route_extractor.rb +23 -6
data/lib/datadog/appsec/api_security/sampler.rb +7 -4
data/lib/datadog/appsec/assets/blocked.html +8 -0
data/lib/datadog/appsec/assets/blocked.json +1 -1
data/lib/datadog/appsec/assets/blocked.text +3 -1
data/lib/datadog/appsec/assets.rb +1 -1
data/lib/datadog/appsec/remote.rb +4 -0
data/lib/datadog/appsec/response.rb +18 -4
data/lib/datadog/core/configuration/components.rb +30 -3
data/lib/datadog/core/configuration/config_helper.rb +1 -1
data/lib/datadog/core/configuration/settings.rb +14 -0
data/lib/datadog/core/configuration/supported_configurations.rb +330 -301
data/lib/datadog/core/ddsketch.rb +0 -2
data/lib/datadog/core/environment/ext.rb +6 -0
data/lib/datadog/core/environment/process.rb +79 -0
data/lib/datadog/core/feature_flags.rb +61 -0
data/lib/datadog/core/remote/client/capabilities.rb +7 -0
data/lib/datadog/core/remote/transport/config.rb +2 -10
data/lib/datadog/core/remote/transport/http/config.rb +9 -9
data/lib/datadog/core/remote/transport/http/negotiation.rb +17 -8
data/lib/datadog/core/remote/transport/http.rb +2 -0
data/lib/datadog/core/remote/transport/negotiation.rb +2 -18
data/lib/datadog/core/remote/worker.rb +25 -37
data/lib/datadog/core/tag_builder.rb +0 -4
data/lib/datadog/core/tag_normalizer.rb +84 -0
data/lib/datadog/core/telemetry/component.rb +7 -3
data/lib/datadog/core/telemetry/event/app_started.rb +52 -49
data/lib/datadog/core/telemetry/event/synth_app_client_configuration_change.rb +1 -1
data/lib/datadog/core/telemetry/logger.rb +2 -2
data/lib/datadog/core/telemetry/logging.rb +2 -8
data/lib/datadog/core/telemetry/transport/http/telemetry.rb +5 -6
data/lib/datadog/core/telemetry/transport/telemetry.rb +1 -2
data/lib/datadog/core/transport/http/client.rb +69 -0
data/lib/datadog/core/utils/array.rb +29 -0
data/lib/datadog/{appsec/api_security → core/utils}/lru_cache.rb +10 -21
data/lib/datadog/core/utils/network.rb +3 -1
data/lib/datadog/core/utils/only_once_successful.rb +6 -2
data/lib/datadog/core/utils.rb +2 -0
data/lib/datadog/data_streams/configuration/settings.rb +49 -0
data/lib/datadog/data_streams/configuration.rb +11 -0
data/lib/datadog/data_streams/ext.rb +11 -0
data/lib/datadog/data_streams/extensions.rb +16 -0
data/lib/datadog/data_streams/pathway_context.rb +169 -0
data/lib/datadog/data_streams/processor.rb +509 -0
data/lib/datadog/data_streams/transport/http/api.rb +33 -0
data/lib/datadog/data_streams/transport/http/client.rb +21 -0
data/lib/datadog/data_streams/transport/http/stats.rb +87 -0
data/lib/datadog/data_streams/transport/http.rb +41 -0
data/lib/datadog/data_streams/transport/stats.rb +60 -0
data/lib/datadog/data_streams.rb +100 -0
data/lib/datadog/di/component.rb +0 -16
data/lib/datadog/di/el/evaluator.rb +1 -1
data/lib/datadog/di/error.rb +4 -0
data/lib/datadog/di/instrumenter.rb +76 -30
data/lib/datadog/di/probe.rb +20 -0
data/lib/datadog/di/probe_manager.rb +10 -2
data/lib/datadog/di/probe_notification_builder.rb +62 -23
data/lib/datadog/di/proc_responder.rb +32 -0
data/lib/datadog/di/transport/diagnostics.rb +2 -2
data/lib/datadog/di/transport/http/diagnostics.rb +2 -4
data/lib/datadog/di/transport/http/input.rb +2 -4
data/lib/datadog/di/transport/http.rb +6 -2
data/lib/datadog/di/transport/input.rb +64 -4
data/lib/datadog/open_feature/component.rb +60 -0
data/lib/datadog/open_feature/configuration.rb +27 -0
data/lib/datadog/open_feature/evaluation_engine.rb +69 -0
data/lib/datadog/open_feature/exposures/batch_builder.rb +32 -0
data/lib/datadog/open_feature/exposures/buffer.rb +43 -0
data/lib/datadog/open_feature/exposures/deduplicator.rb +30 -0
data/lib/datadog/open_feature/exposures/event.rb +60 -0
data/lib/datadog/open_feature/exposures/reporter.rb +40 -0
data/lib/datadog/open_feature/exposures/worker.rb +116 -0
data/lib/datadog/open_feature/ext.rb +14 -0
data/lib/datadog/open_feature/native_evaluator.rb +38 -0
data/lib/datadog/open_feature/noop_evaluator.rb +26 -0
data/lib/datadog/open_feature/provider.rb +141 -0
data/lib/datadog/open_feature/remote.rb +74 -0
data/lib/datadog/open_feature/resolution_details.rb +35 -0
data/lib/datadog/open_feature/transport.rb +72 -0
data/lib/datadog/open_feature.rb +19 -0
data/lib/datadog/opentelemetry/configuration/settings.rb +159 -0
data/lib/datadog/opentelemetry/metrics.rb +110 -0
data/lib/datadog/opentelemetry/sdk/configurator.rb +25 -1
data/lib/datadog/opentelemetry/sdk/metrics_exporter.rb +38 -0
data/lib/datadog/opentelemetry.rb +3 -0
data/lib/datadog/profiling/collectors/code_provenance.rb +15 -6
data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb +1 -1
data/lib/datadog/profiling/collectors/idle_sampling_helper.rb +1 -1
data/lib/datadog/profiling/profiler.rb +4 -0
data/lib/datadog/profiling/tag_builder.rb +36 -3
data/lib/datadog/profiling.rb +1 -2
data/lib/datadog/single_step_instrument.rb +1 -1
data/lib/datadog/tracing/configuration/ext.rb +9 -0
data/lib/datadog/tracing/configuration/settings.rb +74 -0
data/lib/datadog/tracing/contrib/action_pack/action_controller/instrumentation.rb +4 -4
data/lib/datadog/tracing/contrib/action_pack/utils.rb +1 -2
data/lib/datadog/tracing/contrib/active_job/log_injection.rb +21 -7
data/lib/datadog/tracing/contrib/active_job/patcher.rb +5 -1
data/lib/datadog/tracing/contrib/aws/instrumentation.rb +4 -2
data/lib/datadog/tracing/contrib/ethon/easy_patch.rb +4 -1
data/lib/datadog/tracing/contrib/excon/configuration/settings.rb +11 -3
data/lib/datadog/tracing/contrib/faraday/configuration/settings.rb +11 -7
data/lib/datadog/tracing/contrib/grape/configuration/settings.rb +7 -3
data/lib/datadog/tracing/contrib/graphql/unified_trace.rb +22 -17
data/lib/datadog/tracing/contrib/http/configuration/settings.rb +11 -3
data/lib/datadog/tracing/contrib/httpclient/configuration/settings.rb +11 -3
data/lib/datadog/tracing/contrib/httprb/configuration/settings.rb +11 -3
data/lib/datadog/tracing/contrib/kafka/instrumentation/consumer.rb +66 -0
data/lib/datadog/tracing/contrib/kafka/instrumentation/producer.rb +66 -0
data/lib/datadog/tracing/contrib/kafka/patcher.rb +14 -0
data/lib/datadog/tracing/contrib/karafka/framework.rb +30 -0
data/lib/datadog/tracing/contrib/karafka/monitor.rb +11 -0
data/lib/datadog/tracing/contrib/karafka/patcher.rb +32 -0
data/lib/datadog/tracing/contrib/rack/middlewares.rb +59 -27
data/lib/datadog/tracing/contrib/rack/route_inference.rb +53 -0
data/lib/datadog/tracing/contrib/rails/middlewares.rb +2 -2
data/lib/datadog/tracing/contrib/rest_client/request_patch.rb +4 -1
data/lib/datadog/tracing/contrib/roda/instrumentation.rb +3 -1
data/lib/datadog/tracing/contrib/sinatra/tracer_middleware.rb +3 -1
data/lib/datadog/tracing/contrib/status_range_matcher.rb +7 -0
data/lib/datadog/tracing/contrib/waterdrop/configuration/settings.rb +27 -0
data/lib/datadog/tracing/contrib/waterdrop/distributed/propagation.rb +48 -0
data/lib/datadog/tracing/contrib/waterdrop/ext.rb +17 -0
data/lib/datadog/tracing/contrib/waterdrop/integration.rb +43 -0
data/lib/datadog/tracing/contrib/waterdrop/middleware.rb +46 -0
data/lib/datadog/tracing/contrib/waterdrop/patcher.rb +46 -0
data/lib/datadog/tracing/contrib/waterdrop/producer.rb +50 -0
data/lib/datadog/tracing/contrib/waterdrop.rb +37 -0
data/lib/datadog/tracing/contrib.rb +1 -0
data/lib/datadog/tracing/metadata/ext.rb +1 -1
data/lib/datadog/tracing/transport/http/client.rb +12 -26
data/lib/datadog/tracing/transport/trace_formatter.rb +11 -0
data/lib/datadog/tracing/transport/traces.rb +3 -5
data/lib/datadog/version.rb +2 -2
data/lib/datadog.rb +2 -0
metadata +78 -15
data/lib/datadog/core/remote/transport/http/client.rb +0 -49
data/lib/datadog/core/telemetry/transport/http/client.rb +0 -49
data/lib/datadog/di/transport/http/client.rb +0 -47

data/lib/datadog/appsec/api_security/sampler.rb CHANGED Viewed

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 require 'zlib'
-require_relative 'lru_cache'
 require_relative 'route_extractor'
 require_relative '../../core/utils/time'
+require_relative '../../core/utils/lru_cache'
 module Datadog
   module AppSec
@@ -37,20 +37,23 @@ module Datadog
         def initialize(sample_delay)
           raise ArgumentError, 'sample_delay must be an Integer' unless sample_delay.is_a?(Integer)
-          @cache = LRUCache.new(MAX_CACHE_SIZE)
+          @cache = Core::Utils::LRUCache.new(MAX_CACHE_SIZE)
           @sample_delay_seconds = sample_delay
         end
         def sample?(request, response)
           return true if @sample_delay_seconds.zero?
+          return false if response.status == 404
-          key = Zlib.crc32("#{request.request_method}#{RouteExtractor.route_pattern(request)}#{response.status}")
+          route_pattern = RouteExtractor.route_pattern(request).to_s
+          key = Zlib.crc32("#{request.request_method}#{route_pattern}#{response.status}")
           current_timestamp = Core::Utils::Time.now.to_i
           cached_timestamp = @cache[key] || 0
           return false if current_timestamp - cached_timestamp <= @sample_delay_seconds
-          @cache.store(key, current_timestamp)
+          @cache[key] = current_timestamp
           true
         end
       end

data/lib/datadog/appsec/assets/blocked.html CHANGED Viewed

@@ -82,12 +82,20 @@
         footer p {
             font-size: 16px
         }
+        .security-response-id {
+            font-size:14px;
+            color:#999;
+            margin-top:20px;
+            font-family:monospace
+        }
     </style>
 </head>
 <body>
     <main>
         <p>Sorry, you cannot access this page. Please contact the customer service team.</p>
+        <p class="security-response-id">Security Response ID: [security_response_id]</p>
     </main>
     <footer>
         <p>Security provided by <a

data/lib/datadog/appsec/assets/blocked.json CHANGED Viewed

	@@ -1 +1 @@
1	- {"errors": [{"title": "You've been blocked", "detail": "Sorry, you cannot access this page. Please contact the customer service team. Security provided by Datadog."}]}
1	+ {"errors":[{"title":"You've been blocked","detail":"Sorry, you cannot access this page. Please contact the customer service team. Security provided by Datadog."}],"security_response_id":"[security_response_id]"}

data/lib/datadog/appsec/assets/blocked.text CHANGED Viewed

@@ -1,5 +1,7 @@
-You've been blocked
+You've been blocked.
 Sorry, you cannot access this page. Please contact the customer service team.
+Security Response ID: [security_response_id]
 Security provided by Datadog.

data/lib/datadog/appsec/assets.rb CHANGED Viewed

@@ -21,7 +21,7 @@ module Datadog
       end
       def blocked(format: :html)
-        (@blocked ||= {})[format] ||= read("blocked.#{format}")
+        (@blocked ||= {})[format] ||= read("blocked.#{format}").freeze
       end
       def path

data/lib/datadog/appsec/remote.rb CHANGED Viewed

@@ -23,6 +23,8 @@ module Datadog
         CAP_ASM_CUSTOM_RULES = 1 << 8
         CAP_ASM_CUSTOM_BLOCKING_RESPONSE = 1 << 9
         CAP_ASM_TRUSTED_IPS = 1 << 10
+        CAP_ASM_PROCESSOR_OVERRIDES = 1 << 16
+        CAP_ASM_CUSTOM_DATA_SCANNERS = 1 << 17
         CAP_ASM_RASP_SSRF = 1 << 23
         CAP_ASM_RASP_SQLI = 1 << 21
         CAP_ASM_AUTO_USER_INSTRUM_MODE = 1 << 31
@@ -43,6 +45,8 @@ module Datadog
           CAP_ASM_CUSTOM_RULES,
           CAP_ASM_CUSTOM_BLOCKING_RESPONSE,
           CAP_ASM_TRUSTED_IPS,
+          CAP_ASM_PROCESSOR_OVERRIDES,
+          CAP_ASM_CUSTOM_DATA_SCANNERS,
           CAP_ASM_RASP_SSRF,
           CAP_ASM_RASP_SQLI,
           CAP_ASM_AUTO_USER_INSTRUM_MODE,

data/lib/datadog/appsec/response.rb CHANGED Viewed

@@ -7,6 +7,8 @@ module Datadog
   module AppSec
     # AppSec response
     class Response
+      SECURITY_RESPONSE_ID_PLACEHOLDER = '[security_response_id]'
       attr_reader :status, :headers, :body
       def initialize(status:, headers: {}, body: [])
@@ -37,16 +39,26 @@ module Datadog
           Response.new(
             status: interrupt_params['status_code']&.to_i || 403,
             headers: {'Content-Type' => content_type},
-            body: [content(content_type)],
+            body: [
+              content(
+                security_response_id: interrupt_params['security_response_id'],
+                content_type: content_type
+              )
+            ],
           )
         end
         def redirect_response(interrupt_params)
           status_code = interrupt_params['status_code'].to_i
+          location = interrupt_params.fetch('location')
+          if (security_response_id = interrupt_params.fetch('security_response_id'))
+            location.gsub!(SECURITY_RESPONSE_ID_PLACEHOLDER, security_response_id)
+          end
           Response.new(
             status: ((status_code >= 300 && status_code < 400) ? status_code : 303),
-            headers: {'Location' => interrupt_params.fetch('location')},
+            headers: {'Location' => location},
             body: [],
           )
         end
@@ -82,16 +94,18 @@ module Datadog
           DEFAULT_CONTENT_TYPE
         end
-        def content(content_type)
+        def content(security_response_id:, content_type:)
           content_format = CONTENT_TYPE_TO_FORMAT[content_type]
           using_default = Datadog.configuration.appsec.block.templates.using_default?(content_format)
-          if using_default
+          template = if using_default
             Datadog::AppSec::Assets.blocked(format: content_format)
           else
             Datadog.configuration.appsec.block.templates.send(content_format)
           end
+          template.gsub(SECURITY_RESPONSE_ID_PLACEHOLDER, security_response_id.to_s)
         end
       end
     end

data/lib/datadog/core/configuration/components.rb CHANGED Viewed

@@ -15,10 +15,12 @@ require_relative '../../tracing/component'
 require_relative '../../profiling/component'
 require_relative '../../appsec/component'
 require_relative '../../di/component'
+require_relative '../../open_feature/component'
 require_relative '../../error_tracking/component'
 require_relative '../crashtracking/component'
 require_relative '../environment/agent_info'
 require_relative '../process_discovery'
+require_relative '../../data_streams/processor'
 module Datadog
   module Core
@@ -75,11 +77,26 @@ module Datadog
             Datadog::Core::Crashtracking::Component.build(settings, agent_settings, logger: logger)
           end
+          def build_data_streams(settings, agent_settings, logger)
+            return unless settings.data_streams.enabled
+            Datadog::DataStreams::Processor.new(
+              interval: settings.data_streams.interval,
+              logger: logger,
+              settings: settings,
+              agent_settings: agent_settings
+            )
+          rescue => e
+            logger.warn("Failed to initialize Data Streams Monitoring: #{e.class}: #{e}")
+            nil
+          end
         end
         attr_reader \
           :health_metrics,
           :settings,
+          :agent_settings,
           :logger,
           :remote,
           :profiler,
@@ -90,7 +107,9 @@ module Datadog
           :error_tracking,
           :dynamic_instrumentation,
           :appsec,
-          :agent_info
+          :agent_info,
+          :data_streams,
+          :open_feature
         def initialize(settings)
           @settings = settings
@@ -102,7 +121,7 @@ module Datadog
           # This agent_settings is intended for use within Core. If you require
           # agent_settings within a product outside of core you should extend
           # the Core resolver from within your product/component's namespace.
-          agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
+          @agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
           # Exposes agent capability information for detection by any components
           @agent_info = Core::Environment::AgentInfo.new(agent_settings, logger: @logger)
@@ -124,8 +143,10 @@ module Datadog
           @runtime_metrics = self.class.build_runtime_metrics_worker(settings, @logger, telemetry)
           @health_metrics = self.class.build_health_metrics(settings, @logger, telemetry)
           @appsec = Datadog::AppSec::Component.build_appsec_component(settings, telemetry: telemetry)
+          @open_feature = OpenFeature::Component.build(settings, agent_settings, logger: @logger, telemetry: telemetry)
           @dynamic_instrumentation = Datadog::DI::Component.build(settings, agent_settings, @logger, telemetry: telemetry)
           @error_tracking = Datadog::ErrorTracking::Component.build(settings, @tracer, @logger)
+          @data_streams = self.class.build_data_streams(settings, agent_settings, @logger)
           @environment_logger_extra[:dynamic_instrumentation_enabled] = !!@dynamic_instrumentation
           # Configure non-privileged components.
@@ -142,7 +163,7 @@ module Datadog
         # Starts up components
         def startup!(settings, old_state: nil)
-          telemetry.start(old_state&.telemetry_enabled?)
+          telemetry.start(old_state&.telemetry_enabled?, components: self)
           if settings.profiling.enabled
             if profiler
@@ -182,6 +203,9 @@ module Datadog
           # Shutdown DI after remote, since remote config triggers DI operations.
           dynamic_instrumentation&.shutdown!
+          # Shutdown OpenFeature component
+          open_feature&.shutdown!
           # Decommission AppSec
           appsec&.shutdown!
@@ -195,6 +219,9 @@ module Datadog
           # Shutdown workers
           runtime_metrics.stop(true, close_metrics: false)
+          # Shutdown Data Streams Monitoring processor
+          data_streams&.stop(true)
           # Shutdown the old metrics, unless they are still being used.
           # (e.g. custom Statsd instances.)
           #

data/lib/datadog/core/configuration/config_helper.rb CHANGED Viewed

@@ -57,7 +57,7 @@ module Datadog
           # Until we've correctly implemented support for datadog-ci-rb, we disable config inversion if ci is enabled.
           if !defined?(::Datadog::CI) &&
               (name.start_with?('DD_', 'OTEL_') || @alias_to_canonical[name]) &&
-              !@supported_configurations[name]
+              !@supported_configurations.include?(name)
             if defined?(@raise_on_unknown_env_var) && @raise_on_unknown_env_var # Only enabled for tests!
               if @alias_to_canonical[name]
                 raise "Please use #{@alias_to_canonical[name]} instead of #{name}. See docs/AccessEnvironmentVariables.md for details."

data/lib/datadog/core/configuration/settings.rb CHANGED Viewed

@@ -12,6 +12,7 @@ require_relative '../remote/ext'
 require_relative '../../profiling/ext'
 require_relative '../../tracing/configuration/settings'
+require_relative '../../opentelemetry/configuration/settings'
 module Datadog
   module Core
@@ -828,6 +829,7 @@ module Datadog
           # Overrides agentless telemetry URL. To be used internally for testing purposes only.
           #
+          # @default `DD_TELEMETRY_AGENTLESS_URL` environment variable, otherwise `nil`.
           # @return [String]
           # @!visibility private
           option :agentless_url_override do |o|
@@ -1003,6 +1005,16 @@ module Datadog
           end
         end
+        # Enable experimental process tags propagation such that payloads like spans contain the process tag.
+        #
+        # @default `DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED` environment variable, otherwise `false`
+        # @return [Boolean]
+        option :experimental_propagate_process_tags_enabled do |o|
+          o.env 'DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED'
+          o.default false
+          o.type :bool
+        end
         # Tracer specific configuration starting with APM (e.g. DD_APM_TRACING_ENABLED).
         # @public_api
         settings :apm do
@@ -1030,6 +1042,8 @@ module Datadog
         # TODO: Tracing should manage its own settings.
         #       Keep this extension here for now to keep things working.
         extend Datadog::Tracing::Configuration::Settings
+        extend Datadog::OpenTelemetry::Configuration::Settings
       end
       # standard:enable Metrics/BlockLength
     end