ddtrace 1.10.1 → 1.11.0.beta1

data/lib/datadog/core/metrics/metric.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module Core
     module Metrics

data/lib/datadog/core/remote/client/capabilities.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require_relative '../../../appsec/remote'
+
+module Datadog
+  module Core
+    module Remote
+      class Client
+        # Capbailities
+        class Capabilities
+          attr_reader :products, :capabilities, :receivers, :base64_capabilities
+
+          def initialize(settings)
+            @capabilities = []
+            @products = []
+            @receivers = []
+
+            register(settings)
+
+            @base64_capabilities = capabilities_to_base64
+          end
+
+          private
+
+          def register(settings)
+            if settings.appsec.enabled
+              register_capabilities(Datadog::AppSec::Remote.capabilities)
+              register_products(Datadog::AppSec::Remote.products)
+              register_receivers(Datadog::AppSec::Remote.receivers)
+            end
+          end
+
+          def register_capabilities(capabilities)
+            @capabilities.concat(capabilities)
+          end
+
+          def register_receivers(receivers)
+            @receivers.concat(receivers)
+          end
+
+          def register_products(products)
+            @products.concat(products)
+          end
+
+          def capabilities_to_base64
+            return '' if capabilities.empty?
+
+            cap_to_hexs = capabilities.reduce(:|).to_s(16).tap { |s| s.size.odd? && s.prepend('0') }.scan(/\h\h/)
+            binary = cap_to_hexs.each_with_object([]) { |hex, acc| acc << hex }.map { |e| e.to_i(16) }.pack('C*')
+
+            Base64.encode64(binary).chomp
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/remote/client.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+require_relative 'configuration'
+require_relative 'dispatcher'
+
+module Datadog
+  module Core
+    module Remote
+      # Client communicates with the agent and sync remote configuration
+      class Client
+        class SyncError < StandardError; end
+
+        attr_reader :transport, :repository, :id, :dispatcher
+
+        def initialize(transport, capabilities, repository: Configuration::Repository.new)
+          @transport = transport
+
+          @repository = repository
+          @id = SecureRandom.uuid
+          @dispatcher = Dispatcher.new
+          @capabilities = capabilities
+
+          @capabilities.receivers.each do |receiver|
+            dispatcher.receivers << receiver
+          end
+        end
+
+        # rubocop:disable Metrics/AbcSize,Metrics/PerceivedComplexity
+        def sync
+          # TODO: Skip sync if no capabilities are registered
+          response = transport.send_config(payload)
+
+          if response.ok?
+            # when response is completely empty, do nothing as in: leave as is
+            if response.empty?
+              Datadog.logger.debug { 'remote: empty response => NOOP' }
+
+              return
+            end
+
+            paths = response.client_configs.map do |path|
+              Configuration::Path.parse(path)
+            end
+
+            targets = Configuration::TargetMap.parse(response.targets)
+
+            contents = Configuration::ContentList.parse(response.target_files)
+
+            # TODO: sometimes it can strangely be so that paths.empty?
+            # TODO: sometimes it can strangely be so that targets.empty?
+
+            changes = repository.transaction do |current, transaction|
+              # paths to be removed: previously applied paths minus ingress paths
+              (current.paths - paths).each { |p| transaction.delete(p) }
+
+              # go through each ingress path
+              paths.each do |path|
+                # match target with path
+                target = targets[path]
+
+                # abort entirely if matching target not found
+                raise SyncError, "no target for path '#{path}'" if target.nil?
+
+                # new paths are not in previously applied paths
+                new = !current.paths.include?(path)
+
+                # updated paths are in previously applied paths
+                # but the content hash changed
+                changed = current.paths.include?(path) && !current.contents.find_content(path, target)
+
+                # skip if unchanged
+                same = !new && !changed
+
+                next if same
+
+                # match content with path and target
+                content = contents.find_content(path, target)
+
+                # abort entirely if matching content not found
+                raise SyncError, "no valid content for target at path '#{path}'" if content.nil?
+
+                # to be added or updated << config
+                # TODO: metadata (hash, version, etc...)
+                transaction.insert(path, target, content) if new
+                transaction.update(path, target, content) if changed
+              end
+
+              # save backend opaque backend state
+              transaction.set(opaque_backend_state: targets.opaque_backend_state)
+              transaction.set(targets_version: targets.version)
+
+              # upon transaction end, new list of applied config + metadata (add, change, remove) will be saved
+              # TODO: also remove stale config (matching removed) from cache (client configs is exhaustive list of paths)
+            end
+
+            if changes.empty?
+              Datadog.logger.debug { 'remote: no changes' }
+            else
+              dispatcher.dispatch(changes, repository)
+            end
+          else
+            raise SyncError, "unexpected transport response: #{response.inspect}"
+          end
+        end
+        # rubocop:enable Metrics/AbcSize,Metrics/PerceivedComplexity
+
+        private
+
+        def payload
+          state = repository.state
+
+          {
+            client: {
+              state: {
+                root_version: state.root_version,
+                targets_version: state.targets_version,
+                config_states: state.config_states,
+                has_error: state.has_error,
+                error: state.error,
+                backend_client_state: state.opaque_backend_state,
+              },
+              id: id,
+              products: @capabilities.products,
+              is_tracer: true,
+              is_agent: false,
+              client_tracer: {
+                runtime_id: Core::Environment::Identity.id,
+                language: Core::Environment::Identity.lang,
+                tracer_version: Core::Environment::Identity.tracer_version,
+                service: Datadog.configuration.service,
+                env: Datadog.configuration.env,
+                # app_version: app_version, # TODO: I don't know what this is
+                tags: [], # TODO: add nice tags!
+              },
+              # base64 is needed otherwise the Go agent fails with an unmarshal error
+              capabilities: @capabilities.base64_capabilities
+            },
+            cached_target_files: state.cached_target_files,
+          }
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/remote/component.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require_relative 'worker'
+require_relative 'client/capabilities'
+require_relative 'client'
+require_relative '../transport/http'
+require_relative '../remote'
+
+module Datadog
+  module Core
+    module Remote
+      # Configures the HTTP transport to communicate with the agent
+      # to fetch and sync the remote configuration
+      class Component
+        BARRIER_TIMEOUT = 1.0 # second
+
+        attr_reader :client
+
+        def initialize(settings, agent_settings)
+          transport_options = {}
+          transport_options[:agent_settings] = agent_settings if agent_settings
+
+          transport_v7 = Datadog::Core::Transport::HTTP.v7(**transport_options.dup)
+
+          capabilities = Client::Capabilities.new(settings)
+
+          @barrier = Barrier.new(BARRIER_TIMEOUT)
+
+          @client = Client.new(transport_v7, capabilities)
+          @worker = Worker.new(interval: settings.remote.poll_interval_seconds) do
+            begin
+              @client.sync
+            rescue StandardError => e
+              Datadog.logger.error do
+                "remote worker error: #{e.class.name} #{e.message} location: #{Array(e.backtrace).first}"
+              end
+
+              # client state is unknown, state might be corrupted
+              @client = Client.new(transport_v7, capabilities)
+
+              # TODO: bail out if too many errors?
+            end
+
+            @barrier.lift
+          end
+        end
+
+        def barrier(kind)
+          @worker.start
+
+          case kind
+          when :once
+            @barrier.wait_once
+          end
+        end
+
+        def shutdown!
+          @worker.stop
+        end
+
+        # Barrier provides a mechanism to fence execution until a condition happens
+        class Barrier
+          def initialize(timeout = nil)
+            @once = false
+            @timeout = timeout
+
+            @mutex = Mutex.new
+            @condition = ConditionVariable.new
+          end
+
+          # Wait for first lift to happen, otherwise don't wait
+          def wait_once(timeout = nil)
+            # TTAS (Test and Test-And-Set) optimisation
+            # Since @once only ever goes from false to true, this is semantically valid
+            return if @once
+
+            begin
+              @mutex.lock
+
+              return if @once
+
+              timeout ||= @timeout
+
+              # rbs/core has a bug, timeout type is incorrectly ?Integer
+              @condition.wait(@mutex, _ = timeout)
+            ensure
+              @mutex.unlock
+            end
+          end
+
+          # Wait for next lift to happen
+          def wait_next(timeout = nil)
+            @mutex.lock
+
+            timeout ||= @timeout
+
+            # rbs/core has a bug, timeout type is incorrectly ?Integer
+            @condition.wait(@mutex, _ = timeout)
+          ensure
+            @mutex.unlock
+          end
+
+          # Release all current waiters
+          def lift
+            @mutex.lock
+
+            @once ||= true
+
+            @condition.broadcast
+          ensure
+            @mutex.unlock
+          end
+        end
+
+        class << self
+          def build(settings, agent_settings)
+            return unless settings.remote.enabled
+
+            transport_options = {}
+            transport_options[:agent_settings] = agent_settings if agent_settings
+
+            transport_root = Datadog::Core::Transport::HTTP.root(**transport_options.dup)
+
+            res = transport_root.send_info
+            if res.ok?
+              if res.endpoints.include?('/v0.7/config')
+                Datadog.logger.debug { 'agent reachable and reports remote configuration endpoint' }
+              else
+                Datadog.logger.error do
+                  'agent reachable but does not report remote configuration endpoint: ' \
+                    'disabling remote configuration for this process.'
+                end
+
+                return
+              end
+            else
+              Datadog.logger.error do
+                'agent unreachable: disabling remote configuration for this process.'
+              end
+
+              return
+            end
+
+            new(settings, agent_settings)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/remote/configuration/content.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require_relative 'path'
+require_relative 'digest'
+
+module Datadog
+  module Core
+    module Remote
+      class Configuration
+        # Content stores the information associated with a specific Configuration::Path
+        class Content
+          class << self
+            def parse(hash)
+              path = Path.parse(hash[:path])
+              data = hash[:content]
+
+              new(path: path, data: data)
+            end
+          end
+
+          attr_reader :path, :data, :hashes
+
+          def initialize(path:, data:)
+            @path = path
+            @data = data
+            @hashes = {}
+          end
+
+          def hexdigest(type)
+            @hashes[type] || compute_and_store_hash(type)
+          end
+
+          def length
+            @length ||= @data.size
+          end
+
+          private
+
+          def compute_and_store_hash(type)
+            @hashes[type] = Digest.hexdigest(type, @data)
+          end
+
+          private_class_method :new
+        end
+
+        # ContentList stores a list of Conetnt instances
+        # It provides convinient methods for finding content base on Configuration::Path and Configuration::Target
+        class ContentList < Array
+          class << self
+            def parse(array)
+              new.concat(array.map { |c| Content.parse(c) })
+            end
+          end
+
+          def find_content(path, target)
+            find { |c| c.path.eql?(path) && target.check(c) }
+          end
+
+          def [](path)
+            find { |c| c.path.eql?(path) }
+          end
+
+          def []=(path, content)
+            map! { |c| c.path.eql?(path) ? content : c }
+          end
+
+          def delete(path)
+            idx = index { |e| e.path.eql?(path) }
+
+            return if idx.nil?
+
+            delete_at(idx)
+          end
+
+          def paths
+            map(&:path).uniq
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/remote/configuration/digest.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module Datadog
+  module Core
+    module Remote
+      class Configuration
+        # Represent a list of Configuration::Digest
+        class DigestList < Array
+          class << self
+            def parse(hash)
+              new.concat(hash.map { |type, hexdigest| Digest.new(type, hexdigest) })
+            end
+          end
+
+          def check(content)
+            map { |digest| digest.check(content) }.reduce(:&)
+          end
+        end
+
+        # Stores and validates different cryptographic hash functions
+        class Digest
+          class InvalidHashTypeError < StandardError; end
+          attr_reader :type, :hexdigest
+
+          DIGEST_CHUNK = 1024
+
+          class << self
+            def hexdigest(type, data)
+              d = case type
+                  when :sha256
+                    ::Digest::SHA256.new
+                  when :sha512
+                    ::Digest::SHA512.new
+                  else
+                    raise InvalidHashTypeError, type
+                  end
+
+              while (buf = data.read(DIGEST_CHUNK))
+                d.update(buf)
+              end
+
+              d.hexdigest
+            ensure
+              data.rewind
+            end
+          end
+
+          def initialize(type, hexdigest)
+            @type = type.to_sym
+            @hexdigest = hexdigest
+          end
+
+          def check(content)
+            content.hexdigest(@type) == hexdigest
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/remote/configuration/path.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Remote
+      class Configuration
+        # Path stores path information
+        class Path
+          class ParseError < StandardError; end
+
+          class << self
+            RE = %r{
+              ^
+              (?<source>
+                datadog/(?<org_id>\d+)
+                |
+                employee
+              )
+              /
+              (?<product>[^/]+)
+              /
+              (?<config_id>[^/]+)
+              /
+              (?<name>[^/]+)
+              $
+            }mx.freeze
+
+            def parse(path)
+              m = RE.match(path)
+
+              raise ParseError, "could not parse: #{path.inspect}" if m.nil?
+
+              org_id = m['org_id'] ? m['org_id'].to_i : nil
+
+              source = m['source']
+              raise ParseError, 'missing source value' unless source
+
+              source = source.delete("/#{org_id}") if org_id
+
+              product = m['product']
+              raise ParseError, 'missing product value' unless product
+
+              config_id = m['config_id']
+              raise ParseError, 'missing config_id value' unless config_id
+
+              name = m['name']
+              raise ParseError, 'missing name value' unless name
+
+              new(source: source, org_id: org_id, product: product, config_id: config_id, name: name)
+            end
+          end
+
+          attr_reader :source, :org_id, :product, :config_id, :name
+
+          def initialize(source:, product:, config_id:, name:, org_id: nil)
+            @source = source
+            @org_id = org_id
+            @product = product
+            @config_id = config_id
+            @name = name
+          end
+
+          private_class_method :new
+
+          def to_s
+            if org_id
+              "#{source}/#{org_id}/#{product}/#{config_id}/#{name}"
+            else
+              "#{source}/#{product}/#{config_id}/#{name}"
+            end
+          end
+
+          def ==(other)
+            return false unless other.is_a?(Path)
+
+            to_s == other.to_s
+          end
+
+          def hash
+            to_s.hash
+          end
+
+          def eql?(other)
+            hash == other.hash
+          end
+        end
+      end
+    end
+  end
+end