datadog 2.0.0.beta1

data/lib/datadog/core/remote/client/capabilities.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require_relative '../../../appsec/remote'
+require_relative '../../../tracing/remote'
+
+module Datadog
+  module Core
+    module Remote
+      class Client
+        # Capabilities
+        class Capabilities
+          attr_reader :products, :capabilities, :receivers, :base64_capabilities
+
+          def initialize(settings)
+            @capabilities = []
+            @products = []
+            @receivers = []
+
+            register(settings)
+
+            @base64_capabilities = capabilities_to_base64
+          end
+
+          private
+
+          def register(settings)
+            if settings.respond_to?(:appsec) && settings.appsec.enabled
+              register_capabilities(Datadog::AppSec::Remote.capabilities)
+              register_products(Datadog::AppSec::Remote.products)
+              register_receivers(Datadog::AppSec::Remote.receivers)
+            end
+
+            register_capabilities(Datadog::Tracing::Remote.capabilities)
+            register_products(Datadog::Tracing::Remote.products)
+            register_receivers(Datadog::Tracing::Remote.receivers)
+          end
+
+          def register_capabilities(capabilities)
+            @capabilities.concat(capabilities)
+          end
+
+          def register_receivers(receivers)
+            @receivers.concat(receivers)
+          end
+
+          def register_products(products)
+            @products.concat(products)
+          end
+
+          def capabilities_to_base64
+            return '' if capabilities.empty?
+
+            cap_to_hexs = capabilities.reduce(:|).to_s(16).tap { |s| s.size.odd? && s.prepend('0') }.scan(/\h\h/)
+            binary = cap_to_hexs.each_with_object([]) { |hex, acc| acc << hex }.map { |e| e.to_i(16) }.pack('C*')
+
+            Base64.encode64(binary).chomp
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/remote/client.rb

@@ -0,0 +1,234 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+require_relative 'configuration'
+require_relative 'dispatcher'
+
+module Datadog
+  module Core
+    module Remote
+      # Client communicates with the agent and sync remote configuration
+      class Client
+        class TransportError < StandardError; end
+        class SyncError < StandardError; end
+
+        attr_reader :transport, :repository, :id, :dispatcher
+
+        def initialize(transport, capabilities, repository: Configuration::Repository.new)
+          @transport = transport
+
+          @repository = repository
+          @id = SecureRandom.uuid
+          @dispatcher = Dispatcher.new
+          @capabilities = capabilities
+
+          @capabilities.receivers.each do |receiver|
+            dispatcher.receivers << receiver
+          end
+        end
+
+        # rubocop:disable Metrics/AbcSize,Metrics/PerceivedComplexity,Metrics/MethodLength,Metrics/CyclomaticComplexity
+        def sync
+          # TODO: Skip sync if no capabilities are registered
+          response = transport.send_config(payload)
+
+          if response.ok?
+            # when response is completely empty, do nothing as in: leave as is
+            if response.empty?
+              Datadog.logger.debug { 'remote: empty response => NOOP' }
+
+              return
+            end
+
+            begin
+              paths = response.client_configs.map do |path|
+                Configuration::Path.parse(path)
+              end
+
+              targets = Configuration::TargetMap.parse(response.targets)
+
+              contents = Configuration::ContentList.parse(response.target_files)
+            rescue Remote::Configuration::Path::ParseError => e
+              raise SyncError, e.message
+            end
+
+            # To make sure steep does not complain
+            return unless paths && targets && contents
+
+            # TODO: sometimes it can strangely be so that paths.empty?
+            # TODO: sometimes it can strangely be so that targets.empty?
+
+            changes = repository.transaction do |current, transaction|
+              # paths to be removed: previously applied paths minus ingress paths
+              (current.paths - paths).each { |p| transaction.delete(p) }
+
+              # go through each ingress path
+              paths.each do |path|
+                # match target with path
+                target = targets[path]
+
+                # abort entirely if matching target not found
+                raise SyncError, "no target for path '#{path}'" if target.nil?
+
+                # new paths are not in previously applied paths
+                new = !current.paths.include?(path)
+
+                # updated paths are in previously applied paths
+                # but the content hash changed
+                changed = current.paths.include?(path) && !current.contents.find_content(path, target)
+
+                # skip if unchanged
+                same = !new && !changed
+
+                next if same
+
+                # match content with path and target
+                content = contents.find_content(path, target)
+
+                # abort entirely if matching content not found
+                raise SyncError, "no valid content for target at path '#{path}'" if content.nil?
+
+                # to be added or updated << config
+                # TODO: metadata (hash, version, etc...)
+                transaction.insert(path, target, content) if new
+                transaction.update(path, target, content) if changed
+              end
+
+              # save backend opaque backend state
+              transaction.set(opaque_backend_state: targets.opaque_backend_state)
+              transaction.set(targets_version: targets.version)
+
+              # upon transaction end, new list of applied config + metadata (add, change, remove) will be saved
+              # TODO: also remove stale config (matching removed) from cache (client configs is exhaustive list of paths)
+            end
+
+            if changes.empty?
+              Datadog.logger.debug { 'remote: no changes' }
+            else
+              dispatcher.dispatch(changes, repository)
+            end
+          elsif response.internal_error?
+            raise TransportError, response.to_s
+          end
+        end
+        # rubocop:enable Metrics/AbcSize,Metrics/PerceivedComplexity,Metrics/MethodLength,Metrics/CyclomaticComplexity
+
+        private
+
+        def payload # rubocop:disable Metrics/MethodLength
+          state = repository.state
+
+          client_tracer_tags = [
+            "platform:#{native_platform}", # native platform
+            # "asm.config.rules:#{}", # TODO: defined|undefined
+            # "asm.config.enabled:#{}", # TODO: true|false|undefined
+            # TODO: Inaccurate if tracing is extracted and version diverges from the datadog gem.
+            #       Update this if this ever occurs.
+            "ruby.tracer.version:#{Core::Environment::Identity.gem_datadog_version}",
+            "ruby.runtime.platform:#{RUBY_PLATFORM}",
+            "ruby.runtime.version:#{RUBY_VERSION}",
+            "ruby.runtime.engine.name:#{RUBY_ENGINE}",
+            "ruby.runtime.engine.version:#{ruby_engine_version}",
+            "ruby.rubygems.platform.local:#{Gem::Platform.local}",
+            "ruby.gem.libddwaf.version:#{gem_spec('libddwaf').version}",
+            "ruby.gem.libddwaf.platform:#{gem_spec('libddwaf').platform}",
+            "ruby.gem.libdatadog.version:#{gem_spec('libdatadog').version}",
+            "ruby.gem.libdatadog.platform:#{gem_spec('libdatadog').platform}",
+          ]
+
+          client_tracer = {
+            runtime_id: Core::Environment::Identity.id,
+            language: Core::Environment::Identity.lang,
+            tracer_version: tracer_version,
+            service: service_name,
+            env: Datadog.configuration.env,
+            tags: client_tracer_tags,
+          }
+
+          app_version = Datadog.configuration.version
+
+          client_tracer[:app_version] = app_version if app_version
+
+          {
+            client: {
+              state: {
+                root_version: state.root_version,
+                targets_version: state.targets_version,
+                config_states: state.config_states,
+                has_error: state.has_error,
+                error: state.error,
+                backend_client_state: state.opaque_backend_state,
+              },
+              id: id,
+              products: @capabilities.products,
+              is_tracer: true,
+              is_agent: false,
+              client_tracer: client_tracer,
+              # base64 is needed otherwise the Go agent fails with an unmarshal error
+              capabilities: @capabilities.base64_capabilities
+            },
+            cached_target_files: state.cached_target_files,
+          }
+        end
+
+        def service_name
+          Datadog.configuration.remote.service || Datadog.configuration.service
+        end
+
+        def tracer_version
+          @tracer_version ||= Core::Environment::Identity.gem_datadog_version_semver2
+        end
+
+        def ruby_engine_version
+          @ruby_engine_version ||= defined?(RUBY_ENGINE_VERSION) ? RUBY_ENGINE_VERSION : RUBY_VERSION
+        end
+
+        def gem_spec(name)
+          (@gem_specs ||= {})[name] ||= ::Gem.loaded_specs[name] || GemSpecificationFallback.new(nil, nil)
+        end
+
+        def native_platform
+          return @native_platform unless @native_platform.nil?
+
+          os = if RUBY_ENGINE == 'jruby'
+                 os_name = java.lang.System.get_property('os.name')
+
+                 case os_name
+                 when /linux/i then 'linux'
+                 when /mac/i   then 'darwin'
+                 else os_name
+                 end
+               else
+                 Gem::Platform.local.os
+               end
+
+          version = if os != 'linux'
+                      nil
+                    elsif RUBY_PLATFORM =~ /linux-(.+)$/
+                      # Old rubygems don't handle non-gnu linux correctly
+                      Regexp.last_match(1)
+                    else
+                      'gnu'
+                    end
+
+          cpu = if RUBY_ENGINE == 'jruby'
+                  os_arch = java.lang.System.get_property('os.arch')
+
+                  case os_arch
+                  when 'amd64' then 'x86_64'
+                  when 'aarch64' then os == 'darwin' ? 'arm64' : 'aarch64'
+                  else os_arch
+                  end
+                else
+                  Gem::Platform.local.cpu
+                end
+
+          @native_platform = [cpu, os, version].compact.join('-')
+        end
+
+        GemSpecificationFallback = _ = Struct.new(:version, :platform) # rubocop:disable Naming/ConstantName
+      end
+    end
+  end
+end

data/lib/datadog/core/remote/component.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+require_relative 'worker'
+require_relative 'client/capabilities'
+require_relative 'client'
+require_relative 'transport/http'
+require_relative '../remote'
+require_relative 'negotiation'
+
+module Datadog
+  module Core
+    module Remote
+      # Configures the HTTP transport to communicate with the agent
+      # to fetch and sync the remote configuration
+      class Component
+        attr_reader :client, :healthy
+
+        def initialize(settings, capabilities, agent_settings)
+          transport_options = {}
+          transport_options[:agent_settings] = agent_settings if agent_settings
+
+          negotiation = Negotiation.new(settings, agent_settings)
+          transport_v7 = Datadog::Core::Remote::Transport::HTTP.v7(**transport_options.dup)
+
+          @barrier = Barrier.new(settings.remote.boot_timeout_seconds)
+
+          @client = Client.new(transport_v7, capabilities)
+          @healthy = false
+          Datadog.logger.debug { "new remote configuration client: #{@client.id}" }
+
+          @worker = Worker.new(interval: settings.remote.poll_interval_seconds) do
+            unless @healthy || negotiation.endpoint?('/v0.7/config')
+              @barrier.lift
+
+              next
+            end
+
+            begin
+              @client.sync
+              @healthy ||= true
+            rescue Client::SyncError => e
+              Datadog.logger.error do
+                "remote worker client sync error: #{e.message} location: #{Array(e.backtrace).first}. skipping sync"
+              end
+            rescue StandardError => e
+              # In case of unexpected errors, reset the negotiation object
+              # given external conditions have changed and the negotiation
+              # negotiation object stores error logging state that should be reset.
+              negotiation = Negotiation.new(settings, agent_settings)
+
+              Datadog.logger.error do
+                "remote worker error: #{e.class.name} #{e.message} location: #{Array(e.backtrace).first}. "\
+                'reseting client state'
+              end
+
+              # client state is unknown, state might be corrupted
+              @client = Client.new(transport_v7, capabilities)
+              @healthy = false
+              Datadog.logger.debug { "new remote configuration client: #{@client.id}" }
+
+              # TODO: bail out if too many errors?
+            end
+
+            @barrier.lift
+          end
+        end
+
+        # Starts the Remote Configuration worker without waiting for first run
+        def start
+          @worker.start
+        end
+
+        # Is the Remote Configuration worker running?
+        def started?
+          @worker.started?
+        end
+
+        # If the worker is not initialized, initialize it.
+        #
+        # Then, waits for one client sync to be executed if `kind` is `:once`.
+        def barrier(_kind)
+          start
+          @barrier.wait_once
+        end
+
+        def shutdown!
+          @worker.stop
+        end
+
+        # Barrier provides a mechanism to fence execution until a condition happens
+        class Barrier
+          def initialize(timeout = nil)
+            @once = false
+            @timeout = timeout
+
+            @mutex = Mutex.new
+            @condition = ConditionVariable.new
+          end
+
+          # Wait for first lift to happen, otherwise don't wait
+          def wait_once(timeout = nil)
+            # TTAS (Test and Test-And-Set) optimisation
+            # Since @once only ever goes from false to true, this is semantically valid
+            return :pass if @once
+
+            begin
+              @mutex.lock
+
+              return :pass if @once
+
+              timeout ||= @timeout
+
+              # - starting with Ruby 3.2, ConditionVariable#wait returns nil on
+              #   timeout and an integer otherwise
+              # - before Ruby 3.2, ConditionVariable returns itself
+              # so we have to rely on @once having been set
+              if RUBY_VERSION >= '3.2'
+                lifted = @condition.wait(@mutex, timeout)
+              else
+                @condition.wait(@mutex, timeout)
+                lifted = @once
+              end
+
+              if lifted
+                :lift
+              else
+                @once = true
+                :timeout
+              end
+            ensure
+              @mutex.unlock
+            end
+          end
+
+          # Release all current waiters
+          def lift
+            @mutex.lock
+
+            @once ||= true
+
+            @condition.broadcast
+          ensure
+            @mutex.unlock
+          end
+        end
+
+        class << self
+          # Because the agent might not be available yet, we can't perform agent-specific checks yet, as they
+          # would prevent remote configuration from ever running.
+          #
+          # Those checks are instead performed inside the worker loop.
+          # This allows users to upgrade their agent while keeping their application running.
+          def build(settings, agent_settings)
+            return unless settings.remote.enabled
+
+            new(settings, Client::Capabilities.new(settings), agent_settings)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/remote/configuration/content.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require_relative 'path'
+require_relative 'digest'
+
+module Datadog
+  module Core
+    module Remote
+      class Configuration
+        # Content stores the information associated with a specific Configuration::Path
+        class Content
+          class << self
+            def parse(hash)
+              path = Path.parse(hash[:path])
+              data = hash[:content]
+
+              new(path: path, data: data)
+            end
+          end
+
+          attr_reader :path, :data, :hashes, :apply_state, :apply_error
+          attr_accessor :version
+
+          def initialize(path:, data:)
+            @path = path
+            @data = data
+            @apply_state = ApplyState::UNACKNOWLEDGED
+            @apply_error = nil
+            @hashes = {}
+            @version = 0
+          end
+
+          def hexdigest(type)
+            @hashes[type] || compute_and_store_hash(type)
+          end
+
+          def length
+            @length ||= @data.size
+          end
+
+          # Sets this configuration as successfully applied.
+          def applied
+            @apply_state = ApplyState::ACKNOWLEDGED
+            @apply_error = nil
+          end
+
+          # Sets this configuration as not successfully applied, with
+          # a message describing the error.
+          def errored(error_message)
+            @apply_state = ApplyState::ERROR
+            @apply_error = error_message
+          end
+
+          module ApplyState
+            # Default state of configurations.
+            # Set until the component consuming the configuration has acknowledged it was applied.
+            UNACKNOWLEDGED = 1
+
+            # Set when the configuration has been successfully applied.
+            ACKNOWLEDGED = 2
+
+            # Set when the configuration has been unsuccessfully applied.
+            ERROR = 3
+          end
+
+          private
+
+          def compute_and_store_hash(type)
+            @hashes[type] = Digest.hexdigest(type, @data)
+          end
+
+          private_class_method :new
+        end
+
+        # ContentList stores a list of Conetnt instances
+        # It provides convinient methods for finding content base on Configuration::Path and Configuration::Target
+        class ContentList < Array
+          class << self
+            def parse(array)
+              new.concat(array.map { |c| Content.parse(c) })
+            end
+          end
+
+          def find_content(path, target)
+            find { |c| c.path.eql?(path) && target.check(c) }
+          end
+
+          def [](path)
+            find { |c| c.path.eql?(path) }
+          end
+
+          def []=(path, content)
+            map! { |c| c.path.eql?(path) ? content : c }
+          end
+
+          def delete(path)
+            idx = index { |e| e.path.eql?(path) }
+
+            return if idx.nil?
+
+            delete_at(idx)
+          end
+
+          def paths
+            map(&:path).uniq
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/remote/configuration/digest.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module Datadog
+  module Core
+    module Remote
+      class Configuration
+        # Represent a list of Configuration::Digest
+        class DigestList < Array
+          class << self
+            def parse(hash)
+              new.concat(hash.map { |type, hexdigest| Digest.new(type, hexdigest) })
+            end
+          end
+
+          def check(content)
+            map { |digest| digest.check(content) }.reduce(:&)
+          end
+        end
+
+        # Stores and validates different cryptographic hash functions
+        class Digest
+          class InvalidHashTypeError < StandardError; end
+          attr_reader :type, :hexdigest
+
+          DIGEST_CHUNK = 1024
+
+          class << self
+            def hexdigest(type, data)
+              d = case type
+                  when :sha256
+                    ::Digest::SHA256.new
+                  when :sha512
+                    ::Digest::SHA512.new
+                  else
+                    raise InvalidHashTypeError, type
+                  end
+
+              while (buf = data.read(DIGEST_CHUNK))
+                d.update(buf)
+              end
+
+              d.hexdigest
+            ensure
+              data.rewind
+            end
+          end
+
+          def initialize(type, hexdigest)
+            @type = type.to_sym
+            @hexdigest = hexdigest
+          end
+
+          def check(content)
+            content.hexdigest(@type) == hexdigest
+          end
+        end
+      end
+    end
+  end
+end