datadog 2.21.0 → 2.38.0

data/lib/datadog/core/remote/transport/config.rb

@@ -2,56 +2,29 @@
 
 require_relative '../../../core/transport/request'
 require_relative '../../../core/transport/parcel'
+require_relative '../../../core/transport/transport'
+require_relative 'http/config'
 
 module Datadog
   module Core
     module Remote
       module Transport
         module Config
-          # Data transfer object for encoded traces
-          class EncodedParcel
-            include Datadog::Core::Transport::Parcel
-
-            def count
-              data.length
-            end
-          end
-
           # Config request
           class Request < Datadog::Core::Transport::Request
           end
 
-          # Config response
-          module Response
-            attr_reader :roots, :targets, :target_files, :client_configs
-
-            def empty?
-              @empty
-            end
-          end
-
           # Config transport
-          class Transport
-            attr_reader :client, :apis, :default_api, :current_api_id, :logger
-
-            def initialize(apis, default_api, logger: Datadog.logger)
-              @apis = apis
-              @logger = logger
-
-              @client = HTTP::Client.new(current_api, logger: logger)
-            end
-
-            ##### there is only one transport! it's negotiation!
+          class Transport < Core::Transport::Transport
             def send_config(payload)
-              json = JSON.dump(payload)
-              parcel = EncodedParcel.new(json)
+              encoder = Core::Encoding::JSONEncoder
+              parcel = Core::Transport::Parcel.new(
+                encoder.encode(payload),
+                content_type: encoder.content_type,
+              )
               request = Request.new(parcel)
 
-              @client.send_config_payload(request)
-            end
-
-            def current_api
-              @apis[HTTP::API::V7]
+              @client.send_request(:config, request)
             end
           end
         end

data/lib/datadog/core/remote/transport/http/config.rb

@@ -2,12 +2,10 @@
 
 require 'json'
 
-require_relative '../config'
-require_relative 'client'
-require_relative '../../../utils/base64'
-require_relative '../../../utils/truncation'
-require_relative '../../../transport/http/response'
 require_relative '../../../transport/http/api/endpoint'
+require_relative '../../../transport/http/response'
+require_relative '../../../utils/base64_codec'
+require_relative '../../../utils/truncation'
 
 module Datadog
   module Core
@@ -18,8 +16,7 @@ module Datadog
           module Config
             # Response from HTTP transport for remote configuration
             class Response
-              include Datadog::Core::Transport::HTTP::Response
-              include Core::Remote::Transport::Config::Response
+              include Core::Transport::HTTP::Response
 
               def initialize(http_response, options = {}) # standard:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
                 super(http_response)
@@ -38,7 +35,7 @@ module Datadog
 
                 # TODO: these fallbacks should be improved
                 roots = payload[:roots] || []
-                targets = payload[:targets] || Datadog::Core::Utils::Base64.strict_encode64('{}')
+                targets = payload[:targets] || Datadog::Core::Utils::Base64Codec.strict_encode64('{}')
                 target_files = payload[:target_files] || []
                 client_configs = payload[:client_configs] || []
 
@@ -48,7 +45,7 @@ module Datadog
                   raise TypeError.new(String, root) unless root.is_a?(String)
 
                   decoded = begin
-                    Datadog::Core::Utils::Base64.strict_decode64(root) # TODO: unprocessed, don't symbolize_names
+                    Datadog::Core::Utils::Base64Codec.strict_decode64(root) # TODO: unprocessed, don't symbolize_names
                   rescue ArgumentError
                     raise DecodeError.new(:roots, root)
                   end
@@ -68,7 +65,7 @@ module Datadog
 
                 @targets = begin
                   decoded = begin
-                    Datadog::Core::Utils::Base64.strict_decode64(targets)
+                    Datadog::Core::Utils::Base64Codec.strict_decode64(targets)
                   rescue ArgumentError
                     raise DecodeError.new(:targets, targets)
                   end
@@ -96,14 +93,14 @@ module Datadog
                   raise TypeError.new(String, raw) unless raw.is_a?(String)
 
                   content = begin
-                    Datadog::Core::Utils::Base64.strict_decode64(raw)
+                    Datadog::Core::Utils::Base64Codec.strict_decode64(raw)
                   rescue ArgumentError
                     raise DecodeError.new(:target_files, raw)
                   end
 
                   {
                     path: h[:path].freeze,
-                    content: StringIO.new(content.freeze),
+                    content: content.freeze,
                   }
                 end.freeze
 
@@ -114,6 +111,12 @@ module Datadog
                 end.freeze
               end
 
+              attr_reader :roots, :targets, :target_files, :client_configs
+
+              def empty?
+                @empty
+              end
+
               def inspect
                 "#{super}, #{
                   {
@@ -174,46 +177,7 @@ module Datadog
               end
             end
 
-            # Extensions for HTTP client
-            module Client
-              def send_config_payload(request)
-                send_request(request) do |api, env|
-                  api.send_config(env)
-                end
-              end
-            end
-
             module API
-              # Extensions for HTTP API Spec
-              module Spec
-                attr_reader :config
-
-                def config=(endpoint)
-                  @config = endpoint
-                end
-
-                def send_config(env, &block)
-                  raise Core::Transport::HTTP::API::Spec::EndpointNotDefinedError.new('config', self) if config.nil?
-
-                  config.call(env, &block)
-                end
-              end
-
-              # Extensions for HTTP API Instance
-              module Instance
-                def send_config(env)
-                  unless spec.is_a?(Config::API::Spec)
-                    raise Core::Transport::HTTP::API::Instance::EndpointNotSupportedError.new(
-                      'config', self
-                    )
-                  end
-
-                  spec.send_config(env) do |request_env|
-                    call(request_env)
-                  end
-                end
-              end
-
               # Endpoint for remote configuration
               class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
                 HEADER_CONTENT_TYPE = 'Content-Type'
@@ -240,10 +204,6 @@ module Datadog
                 end
               end
             end
-
-            # Add remote configuration behavior to transport components
-            ###### overrides send_payload! which calls send_<endpoint>! kills any other possible endpoint!
-            HTTP::Client.include(Config::Client)
           end
         end
       end

data/lib/datadog/core/remote/transport/http/negotiation.rb

@@ -2,8 +2,7 @@
 
 require 'json'
 
-require_relative '../negotiation'
-require_relative 'client'
+require_relative '../../../transport/http/client'
 require_relative '../../../transport/http/response'
 require_relative '../../../transport/http/api/endpoint'
 
@@ -17,7 +16,6 @@ module Datadog
             # Response from HTTP transport for agent feature negotiation
             class Response
               include Datadog::Core::Transport::HTTP::Response
-              include Core::Remote::Transport::Negotiation::Response
 
               def initialize(http_response, options = {})
                 super(http_response)
@@ -29,48 +27,23 @@ module Datadog
                 @config = options[:config]
                 @span_events = options[:span_events]
               end
-            end
 
-            # Extensions for HTTP client
-            module Client
-              def send_info_payload(request)
-                send_request(request) do |api, env|
-                  api.send_info(env)
-                end
-              end
+              # @!attribute [r] version
+              #   The version of the agent.
+              #   @return [String]
+              # @!attribute [r] endpoints
+              #   The HTTP endpoints the agent supports.
+              #   @return [Array<String>]
+              # @!attribute [r] config
+              #   The agent configuration. These are configured by the user when starting the agent, as well as any defaults.
+              #   @return [Hash]
+              # @!attribute [r] span_events
+              #   Whether the agent supports the top-level span events field in flushed spans.
+              #   @return [Boolean,nil]
+              attr_reader :version, :endpoints, :config, :span_events
             end
 
             module API
-              # Extensions for HTTP API Spec
-              module Spec
-                attr_reader :info
-
-                def info=(endpoint)
-                  @info = endpoint
-                end
-
-                def send_info(env, &block)
-                  raise Core::Transport::HTTP::API::Spec::EndpointNotDefinedError.new('info', self) if info.nil?
-
-                  info.call(env, &block)
-                end
-              end
-
-              # Extensions for HTTP API Instance
-              module Instance
-                def send_info(env)
-                  unless spec.is_a?(Negotiation::API::Spec)
-                    raise Core::Transport::HTTP::API::Instance::EndpointNotSupportedError.new(
-                      'info', self
-                    )
-                  end
-
-                  spec.send_info(env) do |request_env|
-                    call(request_env)
-                  end
-                end
-              end
-
               # Endpoint for negotiation
               class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
                 def initialize(path)
@@ -92,9 +65,6 @@ module Datadog
                 end
               end
             end
-
-            # Add negotiation behavior to transport components
-            HTTP::Client.include(Negotiation::Client)
           end
         end
       end

data/lib/datadog/core/remote/transport/http.rb

@@ -4,16 +4,8 @@ require_relative '../../environment/container'
 require_relative '../../environment/ext'
 require_relative '../../transport/ext'
 require_relative '../../transport/http'
-
-# TODO: Improve negotiation to allow per endpoint selection
-#
-# Since endpoint negotiation happens at the `API::Spec` level there can not be
-# a mix of endpoints at various versions or versionless without describing all
-# the possible combinations as specs. See http/api.
-#
-# Below should be:
-# require_relative '../../transport/http/api'
-require_relative 'http/api'
+require_relative 'config'
+require_relative 'negotiation'
 
 # TODO: Decouple transport/http
 #
@@ -27,26 +19,30 @@ module Datadog
       module Transport
         # Namespace for HTTP transport components
         module HTTP
+          ROOT = Negotiation::API::Endpoint.new(
+            '/info',
+          )
+
+          V7 = Config::API::Endpoint.new(
+            '/v0.7/config',
+            Core::Encoding::JSONEncoder,
+          )
+
           module_function
 
           # Builds a new Transport::HTTP::Client with default settings
           # Pass a block to override any settings.
           def root(
             agent_settings:,
-            logger: Datadog.logger,
-            api_version: nil,
+            logger:,
             headers: nil
           )
             Core::Transport::HTTP.build(
-              api_instance_class: API::Instance,
               agent_settings: agent_settings,
               logger: logger,
-              api_version: api_version,
               headers: headers
             ) do |transport|
-              apis = API.defaults
-
-              transport.api API::ROOT, apis[API::ROOT]
+              transport.api 'root', ROOT
 
               # Call block to apply any customization, if provided
               yield(transport) if block_given?
@@ -57,20 +53,15 @@ module Datadog
           # Pass a block to override any settings.
           def v7(
             agent_settings:,
-            logger: Datadog.logger,
-            api_version: nil,
+            logger:,
             headers: nil
           )
             Core::Transport::HTTP.build(
-              api_instance_class: API::Instance,
               agent_settings: agent_settings,
               logger: logger,
-              api_version: api_version,
               headers: headers
             ) do |transport|
-              apis = API.defaults
-
-              transport.api API::V7, apis[API::V7]
+              transport.api 'v7', V7
 
               # Call block to apply any customization, if provided
               yield(transport) if block_given?

data/lib/datadog/core/remote/transport/negotiation.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require_relative '../../../core/transport/request'
+require_relative '../../../core/transport/transport'
+require_relative 'http/negotiation'
 
 # TODO: Resolve conceptual conundrum
 #
@@ -30,42 +32,15 @@ module Datadog
           class Request < Datadog::Core::Transport::Request
           end
 
-          # Negotiation response
-          module Response
-            # @!attribute [r] version
-            #   The version of the agent.
-            #   @return [String]
-            # @!attribute [r] endpoints
-            #   The HTTP endpoints the agent supports.
-            #   @return [Array<String>]
-            # @!attribute [r] config
-            #   The agent configuration. These are configured by the user when starting the agent, as well as any defaults.
-            #   @return [Hash]
-            # @!attribute [r] span_events
-            #   Whether the agent supports the top-level span events field in flushed spans.
-            #   @return [Boolean,nil]
-            attr_reader :version, :endpoints, :config, :span_events
-          end
-
           # Negotiation transport
-          class Transport
-            attr_reader :client, :apis, :default_api, :current_api_id, :logger
-
-            def initialize(apis, default_api, logger: Datadog.logger)
-              @apis = apis
-              @logger = logger
-
-              @client = HTTP::Client.new(current_api, logger: logger)
-            end
-
-            def send_info
+          class Transport < Core::Transport::Transport
+            # TODO steep is complaining about this method, I tried to make
+            # it work but between module inclusions and steep diagnostics
+            # I don't understand what the problem is or what the steep wants.
+            def send_info # steep:ignore
               request = Request.new
 
-              @client.send_info_payload(request)
-            end
-
-            def current_api
-              @apis[HTTP::API::ROOT]
+              @client.send_request(:info, request)
             end
           end
         end

data/lib/datadog/core/remote/worker.rb

@@ -23,51 +23,47 @@ module Datadog
         attr_reader :logger
 
         def start
-          logger.debug { 'remote worker starting' }
+          logger.debug { "remote worker starting (pid: #{Process.pid})" }
 
-          acquire_lock
+          @mutex.synchronize do
+            if @stopped
+              logger.debug('remote worker: refusing to restart after previous stop')
+              return
+            end
 
-          if @stopped
-            logger.debug('remote worker: refusing to restart after previous stop')
-            return
-          end
-
-          return if @starting || @started
+            return if @starting || @started
 
-          @starting = true
+            @starting = true
 
-          thread = Thread.new { poll(@interval) }
-          thread.name = self.class.name
-          thread.thread_variable_set(:fork_safe, true)
-          @thr = thread
+            thread = Thread.new { poll(@interval) }
+            thread.name = self.class.name
+            thread.thread_variable_set(:fork_safe, true)
+            @thr = thread
 
-          @started = true
-          @starting = false
+            @started = true
+            @starting = false
+          end
 
           logger.debug { 'remote worker started' }
-        ensure
-          release_lock
         end
 
         def stop
-          logger.debug { 'remote worker stopping' }
+          logger.debug { "remote worker stopping (pid: #{Process.pid})" }
 
-          acquire_lock
+          @mutex.synchronize do
+            thread = @thr
 
-          thread = @thr
+            if thread
+              thread.kill
+              thread.join
+            end
 
-          if thread
-            thread.kill
-            thread.join
+            @started = false
+            @thr = nil
+            @stopped = true
           end
 
-          @started = false
-          @thr = nil
-          @stopped = true
-
           logger.debug { 'remote worker stopped' }
-        ensure
-          release_lock
         end
 
         def started?
@@ -76,14 +72,6 @@ module Datadog
 
         private
 
-        def acquire_lock
-          @mutex.lock
-        end
-
-        def release_lock
-          @mutex.unlock
-        end
-
         def poll(interval)
           loop do
             break unless @mutex.synchronize { @starting || @started }

data/lib/datadog/core/runtime/metrics.rb

@@ -8,6 +8,7 @@ require_relative '../environment/gc'
 require_relative '../environment/thread_count'
 require_relative '../environment/vm_cache'
 require_relative '../environment/yjit'
+require_relative '../environment/process'
 
 module Datadog
   module Core
@@ -24,6 +25,8 @@ module Datadog
 
           # Initialize the collection of runtime-id
           @runtime_id_enabled = options.fetch(:experimental_runtime_id_enabled, false)
+
+          @process_tags_enabled = options.fetch(:experimental_propagate_process_tags_enabled)
         end
 
         # Associate service with runtime metrics
@@ -97,7 +100,7 @@ module Datadog
         def try_flush
           yield
         rescue => e
-          Datadog.logger.warn("Error while sending runtime metric. Cause: #{e.class.name} #{e.message}")
+          Datadog.logger.warn("Error while sending runtime metric. Cause: #{e.class}: #{e.message}")
         end
 
         def default_metric_options
@@ -111,6 +114,11 @@ module Datadog
 
             # Add runtime-id dynamically because it might change during runtime.
             options[:tags].concat(["runtime-id:#{Core::Environment::Identity.id}"]) if @runtime_id_enabled
+
+            # Add process tags when enabled
+            if @process_tags_enabled
+              options[:tags].concat(Core::Environment::Process.tags)
+            end
           end
         end
 
@@ -119,7 +127,8 @@ module Datadog
         attr_reader \
           :service_tags,
           :services,
-          :runtime_id_enabled
+          :runtime_id_enabled,
+          :process_tags_enabled
 
         def compile_service_tags!
           @service_tags = services.to_a.collect do |service|

data/lib/datadog/core/semaphore.rb

@@ -20,10 +20,7 @@ module Datadog
 
       def wait(timeout = nil)
         wake_lock.synchronize do
-          # steep specifies that the second argument to wait is of type
-          # ::Time::_Timeout which for some reason is not Numeric and is not
-          # castable from Numeric.
-          wake.wait(wake_lock, timeout) # steep:ignore
+          wake.wait(wake_lock, timeout)
         end
       end
 

data/lib/datadog/core/tag_builder.rb

@@ -8,10 +8,6 @@ module Datadog
   module Core
     # This module builds a hash of tags.
     #
-    # When changing or adding the tags, make sure they are kept in sync with
-    # https://docs.google.com/spreadsheets/d/1LOGMf4c4Avbtn36uZ2SWvhIGKRPLM1BoWkUP4JYj7hA/
-    # (Datadog internal link).
-    #
     # @api private
     module TagBuilder
       def self.fixed_environment_tags

data/lib/datadog/core/tag_normalizer.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require_relative 'utils'
+
+module Datadog
+  module Core
+    # @api private
+    module TagNormalizer
+      # Normalization logic used for tag keys and values that the Trace Agent has for traces
+      # Useful for ensuring that tag keys and values are normalized consistently
+      # An use case for now is Process Tags which need to be sent across various intakes (profiling, tracing, etc.) consistently
+
+      module_function
+
+      INVALID_TAG_CHARACTERS = %r{[^\p{L}0-9_\-:./]}
+      LEADING_INVALID_CHARS_NO_DIGITS = %r{\A[^\p{L}:]++}
+      LEADING_INVALID_CHARS_WITH_DIGITS = %r{\A[^\p{L}0-9:./]++}
+      MAX_BYTE_SIZE = 200 # Represents the general max tag length
+      MAX_PROCESS_VALUE_BYTE_SIZE = 100 # Represents the max tag length for process tags
+      VALID_ASCII_TAG = %r{\A[a-z:][a-z0-9:./-]*\z}
+
+      # Based on https://github.com/DataDog/datadog-agent/blob/45799c842bbd216bcda208737f9f11cade6fdd95/pkg/trace/traceutil/normalize.go#L131
+      # Specifically for general normalization:
+      # - Must be valid UTF-8
+      # - Invalid characters are replaced with an underscore
+      # - Leading non-letter characters are removed but colons are kept
+      # - Trailing non-letter characters are removed
+      # - Trailing underscores are removed
+      # - Consecutive underscores are merged into a single underscore
+      # - Maximum length is 200 characters
+      # If it's a tag value, allow it to start with a digit
+      # @param original_value [String] The original string
+      # @param remove_digit_start_char [Boolean] - whether to remove the leading digit (currently only used for tag values)
+      # @return [String] The normalized string
+      def self.normalize(original_value, remove_digit_start_char: false)
+        # DEV-3.0: Ideally this encode call should be replaced with Datadog::Core::Utils.utf8_encode once it
+        #          is safe to modify the default behavior.
+        value = original_value.to_s.encode('UTF-8', invalid: :replace, undef: :replace)
+        value.strip!
+        return "" if value.empty?
+
+        return value if value.bytesize <= MAX_BYTE_SIZE &&
+          value.match?(VALID_ASCII_TAG)
+
+        if value.bytesize > MAX_BYTE_SIZE
+          value = value.byteslice(0, MAX_BYTE_SIZE)
+          value.scrub!("")
+        end
+
+        value.downcase!
+        value.gsub!(INVALID_TAG_CHARACTERS, '_')
+
+        # The Trace Agent allows tag values to start with a number so this logic is here too
+        leading_invalid_regex = remove_digit_start_char ? LEADING_INVALID_CHARS_NO_DIGITS : LEADING_INVALID_CHARS_WITH_DIGITS
+        value.sub!(leading_invalid_regex, "")
+
+        value.squeeze!('_') if value.include?('__')
+        value.delete_suffix!('_')
+
+        value
+      end
+
+      # Process tags values follow an additional piece of normalization:
+      # - must not be more than 100 bytes
+      # - and must not contain colons
+      # @param value [String] The original string
+      # @return [String] The normalized string
+      def self.normalize_process_value(value)
+        value = normalize(value)
+        return value if value.empty?
+
+        value.tr!(':', '_')
+        value.squeeze!('_') if value.include?('__')
+
+        if value.bytesize > MAX_PROCESS_VALUE_BYTE_SIZE
+          value = value.byteslice(0, MAX_PROCESS_VALUE_BYTE_SIZE) || value
+          value.scrub!("")
+        end
+
+        value
+      end
+    end
+  end
+end