ddtrace 1.4.2 → 1.19.0

data/lib/datadog/core/remote/client.rb

@@ -0,0 +1,232 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+require_relative 'configuration'
+require_relative 'dispatcher'
+
+module Datadog
+  module Core
+    module Remote
+      # Client communicates with the agent and sync remote configuration
+      class Client
+        class TransportError < StandardError; end
+        class SyncError < StandardError; end
+
+        attr_reader :transport, :repository, :id, :dispatcher
+
+        def initialize(transport, capabilities, repository: Configuration::Repository.new)
+          @transport = transport
+
+          @repository = repository
+          @id = SecureRandom.uuid
+          @dispatcher = Dispatcher.new
+          @capabilities = capabilities
+
+          @capabilities.receivers.each do |receiver|
+            dispatcher.receivers << receiver
+          end
+        end
+
+        # rubocop:disable Metrics/AbcSize,Metrics/PerceivedComplexity,Metrics/MethodLength,Metrics/CyclomaticComplexity
+        def sync
+          # TODO: Skip sync if no capabilities are registered
+          response = transport.send_config(payload)
+
+          if response.ok?
+            # when response is completely empty, do nothing as in: leave as is
+            if response.empty?
+              Datadog.logger.debug { 'remote: empty response => NOOP' }
+
+              return
+            end
+
+            begin
+              paths = response.client_configs.map do |path|
+                Configuration::Path.parse(path)
+              end
+
+              targets = Configuration::TargetMap.parse(response.targets)
+
+              contents = Configuration::ContentList.parse(response.target_files)
+            rescue Remote::Configuration::Path::ParseError => e
+              raise SyncError, e.message
+            end
+
+            # To make sure steep does not complain
+            return unless paths && targets && contents
+
+            # TODO: sometimes it can strangely be so that paths.empty?
+            # TODO: sometimes it can strangely be so that targets.empty?
+
+            changes = repository.transaction do |current, transaction|
+              # paths to be removed: previously applied paths minus ingress paths
+              (current.paths - paths).each { |p| transaction.delete(p) }
+
+              # go through each ingress path
+              paths.each do |path|
+                # match target with path
+                target = targets[path]
+
+                # abort entirely if matching target not found
+                raise SyncError, "no target for path '#{path}'" if target.nil?
+
+                # new paths are not in previously applied paths
+                new = !current.paths.include?(path)
+
+                # updated paths are in previously applied paths
+                # but the content hash changed
+                changed = current.paths.include?(path) && !current.contents.find_content(path, target)
+
+                # skip if unchanged
+                same = !new && !changed
+
+                next if same
+
+                # match content with path and target
+                content = contents.find_content(path, target)
+
+                # abort entirely if matching content not found
+                raise SyncError, "no valid content for target at path '#{path}'" if content.nil?
+
+                # to be added or updated << config
+                # TODO: metadata (hash, version, etc...)
+                transaction.insert(path, target, content) if new
+                transaction.update(path, target, content) if changed
+              end
+
+              # save backend opaque backend state
+              transaction.set(opaque_backend_state: targets.opaque_backend_state)
+              transaction.set(targets_version: targets.version)
+
+              # upon transaction end, new list of applied config + metadata (add, change, remove) will be saved
+              # TODO: also remove stale config (matching removed) from cache (client configs is exhaustive list of paths)
+            end
+
+            if changes.empty?
+              Datadog.logger.debug { 'remote: no changes' }
+            else
+              dispatcher.dispatch(changes, repository)
+            end
+          elsif response.internal_error?
+            raise TransportError, response.to_s
+          end
+        end
+        # rubocop:enable Metrics/AbcSize,Metrics/PerceivedComplexity,Metrics/MethodLength,Metrics/CyclomaticComplexity
+
+        private
+
+        def payload # rubocop:disable Metrics/MethodLength
+          state = repository.state
+
+          client_tracer_tags = [
+            "platform:#{native_platform}", # native platform
+            # "asm.config.rules:#{}", # TODO: defined|undefined
+            # "asm.config.enabled:#{}", # TODO: true|false|undefined
+            "ruby.tracer.version:#{Core::Environment::Identity.tracer_version}",
+            "ruby.runtime.platform:#{RUBY_PLATFORM}",
+            "ruby.runtime.version:#{RUBY_VERSION}",
+            "ruby.runtime.engine.name:#{RUBY_ENGINE}",
+            "ruby.runtime.engine.version:#{ruby_engine_version}",
+            "ruby.rubygems.platform.local:#{Gem::Platform.local}",
+            "ruby.gem.libddwaf.version:#{gem_spec('libddwaf').version}",
+            "ruby.gem.libddwaf.platform:#{gem_spec('libddwaf').platform}",
+            "ruby.gem.libdatadog.version:#{gem_spec('libdatadog').version}",
+            "ruby.gem.libdatadog.platform:#{gem_spec('libdatadog').platform}",
+          ]
+
+          client_tracer = {
+            runtime_id: Core::Environment::Identity.id,
+            language: Core::Environment::Identity.lang,
+            tracer_version: tracer_version_semver2,
+            service: service_name,
+            env: Datadog.configuration.env,
+            tags: client_tracer_tags,
+          }
+
+          app_version = Datadog.configuration.version
+
+          client_tracer[:app_version] = app_version if app_version
+
+          {
+            client: {
+              state: {
+                root_version: state.root_version,
+                targets_version: state.targets_version,
+                config_states: state.config_states,
+                has_error: state.has_error,
+                error: state.error,
+                backend_client_state: state.opaque_backend_state,
+              },
+              id: id,
+              products: @capabilities.products,
+              is_tracer: true,
+              is_agent: false,
+              client_tracer: client_tracer,
+              # base64 is needed otherwise the Go agent fails with an unmarshal error
+              capabilities: @capabilities.base64_capabilities
+            },
+            cached_target_files: state.cached_target_files,
+          }
+        end
+
+        def service_name
+          Datadog.configuration.remote.service || Datadog.configuration.service
+        end
+
+        def tracer_version_semver2
+          @tracer_version_semver2 ||= Core::Environment::Identity.tracer_version_semver2
+        end
+
+        def ruby_engine_version
+          @ruby_engine_version ||= defined?(RUBY_ENGINE_VERSION) ? RUBY_ENGINE_VERSION : RUBY_VERSION
+        end
+
+        def gem_spec(name)
+          (@gem_specs ||= {})[name] ||= ::Gem.loaded_specs[name] || GemSpecificationFallback.new(nil, nil)
+        end
+
+        def native_platform
+          return @native_platform unless @native_platform.nil?
+
+          os = if RUBY_ENGINE == 'jruby'
+                 os_name = java.lang.System.get_property('os.name')
+
+                 case os_name
+                 when /linux/i then 'linux'
+                 when /mac/i   then 'darwin'
+                 else os_name
+                 end
+               else
+                 Gem::Platform.local.os
+               end
+
+          version = if os != 'linux'
+                      nil
+                    elsif RUBY_PLATFORM =~ /linux-(.+)$/
+                      # Old rubygems don't handle non-gnu linux correctly
+                      Regexp.last_match(1)
+                    else
+                      'gnu'
+                    end
+
+          cpu = if RUBY_ENGINE == 'jruby'
+                  os_arch = java.lang.System.get_property('os.arch')
+
+                  case os_arch
+                  when 'amd64' then 'x86_64'
+                  when 'aarch64' then os == 'darwin' ? 'arm64' : 'aarch64'
+                  else os_arch
+                  end
+                else
+                  Gem::Platform.local.cpu
+                end
+
+          @native_platform = [cpu, os, version].compact.join('-')
+        end
+
+        GemSpecificationFallback = _ = Struct.new(:version, :platform) # rubocop:disable Naming/ConstantName
+      end
+    end
+  end
+end

data/lib/datadog/core/remote/component.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require_relative 'worker'
+require_relative 'client/capabilities'
+require_relative 'client'
+require_relative 'transport/http'
+require_relative '../remote'
+require_relative 'negotiation'
+
+module Datadog
+  module Core
+    module Remote
+      # Configures the HTTP transport to communicate with the agent
+      # to fetch and sync the remote configuration
+      class Component
+        BARRIER_TIMEOUT = 1.0 # second
+
+        attr_reader :client
+
+        def initialize(settings, capabilities, agent_settings)
+          transport_options = {}
+          transport_options[:agent_settings] = agent_settings if agent_settings
+
+          negotiation = Negotiation.new(settings, agent_settings)
+          transport_v7 = Datadog::Core::Remote::Transport::HTTP.v7(**transport_options.dup)
+
+          @barrier = Barrier.new(BARRIER_TIMEOUT)
+
+          @client = Client.new(transport_v7, capabilities)
+          healthy = false
+          Datadog.logger.debug { "new remote configuration client: #{@client.id}" }
+
+          @worker = Worker.new(interval: settings.remote.poll_interval_seconds) do
+            unless healthy || negotiation.endpoint?('/v0.7/config')
+              @barrier.lift
+
+              next
+            end
+
+            begin
+              @client.sync
+              healthy ||= true
+            rescue Client::SyncError => e
+              Datadog.logger.error do
+                "remote worker client sync error: #{e.message} location: #{Array(e.backtrace).first}. skipping sync"
+              end
+            rescue StandardError => e
+              # In case of unexpected errors, reset the negotiation object
+              # given external conditions have changed and the negotiation
+              # negotiation object stores error logging state that should be reset.
+              negotiation = Negotiation.new(settings, agent_settings)
+
+              Datadog.logger.error do
+                "remote worker error: #{e.class.name} #{e.message} location: #{Array(e.backtrace).first}. "\
+                'reseting client state'
+              end
+
+              # client state is unknown, state might be corrupted
+              @client = Client.new(transport_v7, capabilities)
+              healthy = false
+              Datadog.logger.debug { "new remote configuration client: #{@client.id}" }
+
+              # TODO: bail out if too many errors?
+            end
+
+            @barrier.lift
+          end
+        end
+
+        # Starts the Remote Configuration worker without waiting for first run
+        def start
+          @worker.start
+        end
+
+        # Is the Remote Configuration worker running?
+        def started?
+          @worker.started?
+        end
+
+        # If the worker is not initialized, initialize it.
+        #
+        # Then, waits for one client sync to be executed if `kind` is `:once`.
+        def barrier(_kind)
+          start
+          @barrier.wait_once
+        end
+
+        def shutdown!
+          @worker.stop
+        end
+
+        # Barrier provides a mechanism to fence execution until a condition happens
+        class Barrier
+          def initialize(timeout = nil)
+            @once = false
+            @timeout = timeout
+
+            @mutex = Mutex.new
+            @condition = ConditionVariable.new
+          end
+
+          # Wait for first lift to happen, otherwise don't wait
+          def wait_once(timeout = nil)
+            # TTAS (Test and Test-And-Set) optimisation
+            # Since @once only ever goes from false to true, this is semantically valid
+            return if @once
+
+            begin
+              @mutex.lock
+
+              return if @once
+
+              timeout ||= @timeout
+
+              # rbs/core has a bug, timeout type is incorrectly ?Integer
+              @condition.wait(@mutex, _ = timeout)
+            ensure
+              @mutex.unlock
+            end
+          end
+
+          # Release all current waiters
+          def lift
+            @mutex.lock
+
+            @once ||= true
+
+            @condition.broadcast
+          ensure
+            @mutex.unlock
+          end
+        end
+
+        class << self
+          # Because the agent might not be available yet, we can't perform agent-specific checks yet, as they
+          # would prevent remote configuration from ever running.
+          #
+          # Those checks are instead performed inside the worker loop.
+          # This allows users to upgrade their agent while keeping their application running.
+          def build(settings, agent_settings)
+            return unless settings.remote.enabled
+
+            new(settings, Client::Capabilities.new(settings), agent_settings)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/remote/configuration/content.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require_relative 'path'
+require_relative 'digest'
+
+module Datadog
+  module Core
+    module Remote
+      class Configuration
+        # Content stores the information associated with a specific Configuration::Path
+        class Content
+          class << self
+            def parse(hash)
+              path = Path.parse(hash[:path])
+              data = hash[:content]
+
+              new(path: path, data: data)
+            end
+          end
+
+          attr_reader :path, :data, :hashes, :apply_state, :apply_error
+          attr_accessor :version
+
+          def initialize(path:, data:)
+            @path = path
+            @data = data
+            @apply_state = ApplyState::UNACKNOWLEDGED
+            @apply_error = nil
+            @hashes = {}
+            @version = 0
+          end
+
+          def hexdigest(type)
+            @hashes[type] || compute_and_store_hash(type)
+          end
+
+          def length
+            @length ||= @data.size
+          end
+
+          # Sets this configuration as successfully applied.
+          def applied
+            @apply_state = ApplyState::ACKNOWLEDGED
+            @apply_error = nil
+          end
+
+          # Sets this configuration as not successfully applied, with
+          # a message describing the error.
+          def errored(error_message)
+            @apply_state = ApplyState::ERROR
+            @apply_error = error_message
+          end
+
+          module ApplyState
+            # Default state of configurations.
+            # Set until the component consuming the configuration has acknowledged it was applied.
+            UNACKNOWLEDGED = 1
+
+            # Set when the configuration has been successfully applied.
+            ACKNOWLEDGED = 2
+
+            # Set when the configuration has been unsuccessfully applied.
+            ERROR = 3
+          end
+
+          private
+
+          def compute_and_store_hash(type)
+            @hashes[type] = Digest.hexdigest(type, @data)
+          end
+
+          private_class_method :new
+        end
+
+        # ContentList stores a list of Conetnt instances
+        # It provides convinient methods for finding content base on Configuration::Path and Configuration::Target
+        class ContentList < Array
+          class << self
+            def parse(array)
+              new.concat(array.map { |c| Content.parse(c) })
+            end
+          end
+
+          def find_content(path, target)
+            find { |c| c.path.eql?(path) && target.check(c) }
+          end
+
+          def [](path)
+            find { |c| c.path.eql?(path) }
+          end
+
+          def []=(path, content)
+            map! { |c| c.path.eql?(path) ? content : c }
+          end
+
+          def delete(path)
+            idx = index { |e| e.path.eql?(path) }
+
+            return if idx.nil?
+
+            delete_at(idx)
+          end
+
+          def paths
+            map(&:path).uniq
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/remote/configuration/digest.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module Datadog
+  module Core
+    module Remote
+      class Configuration
+        # Represent a list of Configuration::Digest
+        class DigestList < Array
+          class << self
+            def parse(hash)
+              new.concat(hash.map { |type, hexdigest| Digest.new(type, hexdigest) })
+            end
+          end
+
+          def check(content)
+            map { |digest| digest.check(content) }.reduce(:&)
+          end
+        end
+
+        # Stores and validates different cryptographic hash functions
+        class Digest
+          class InvalidHashTypeError < StandardError; end
+          attr_reader :type, :hexdigest
+
+          DIGEST_CHUNK = 1024
+
+          class << self
+            def hexdigest(type, data)
+              d = case type
+                  when :sha256
+                    ::Digest::SHA256.new
+                  when :sha512
+                    ::Digest::SHA512.new
+                  else
+                    raise InvalidHashTypeError, type
+                  end
+
+              while (buf = data.read(DIGEST_CHUNK))
+                d.update(buf)
+              end
+
+              d.hexdigest
+            ensure
+              data.rewind
+            end
+          end
+
+          def initialize(type, hexdigest)
+            @type = type.to_sym
+            @hexdigest = hexdigest
+          end
+
+          def check(content)
+            content.hexdigest(@type) == hexdigest
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/remote/configuration/path.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Remote
+      class Configuration
+        # Path stores path information
+        class Path
+          class ParseError < StandardError; end
+
+          class << self
+            RE = %r{
+              ^
+              (?<source>
+                datadog/(?<org_id>\d+)
+                |
+                employee
+              )
+              /
+              (?<product>[^/]+)
+              /
+              (?<config_id>[^/]+)
+              /
+              (?<name>[^/]+)
+              $
+            }mx.freeze
+
+            def parse(path)
+              m = RE.match(path)
+
+              raise ParseError, "could not parse: #{path.inspect}" if m.nil?
+
+              org_id = m['org_id'] ? m['org_id'].to_i : nil
+
+              source = m['source']
+              raise ParseError, 'missing source value' unless source
+
+              source = source.delete("/#{org_id}") if org_id
+
+              product = m['product']
+              raise ParseError, 'missing product value' unless product
+
+              config_id = m['config_id']
+              raise ParseError, 'missing config_id value' unless config_id
+
+              name = m['name']
+              raise ParseError, 'missing name value' unless name
+
+              new(source: source, org_id: org_id, product: product, config_id: config_id, name: name)
+            end
+          end
+
+          attr_reader :source, :org_id, :product, :config_id, :name
+
+          def initialize(source:, product:, config_id:, name:, org_id: nil)
+            @source = source
+            @org_id = org_id
+            @product = product
+            @config_id = config_id
+            @name = name
+          end
+
+          private_class_method :new
+
+          def to_s
+            if org_id
+              "#{source}/#{org_id}/#{product}/#{config_id}/#{name}"
+            else
+              "#{source}/#{product}/#{config_id}/#{name}"
+            end
+          end
+
+          def ==(other)
+            return false unless other.is_a?(Path)
+
+            to_s == other.to_s
+          end
+
+          def hash
+            to_s.hash
+          end
+
+          def eql?(other)
+            hash == other.hash
+          end
+        end
+      end
+    end
+  end
+end