ddtrace 1.10.1 → 1.11.0

data/lib/datadog/core/remote/component.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require_relative 'worker'
+require_relative 'client/capabilities'
+require_relative 'client'
+require_relative '../transport/http'
+require_relative '../remote'
+require_relative 'negotiation'
+
+module Datadog
+  module Core
+    module Remote
+      # Configures the HTTP transport to communicate with the agent
+      # to fetch and sync the remote configuration
+      class Component
+        BARRIER_TIMEOUT = 1.0 # second
+
+        attr_reader :client
+
+        def initialize(settings, capabilities, agent_settings)
+          transport_options = {}
+          transport_options[:agent_settings] = agent_settings if agent_settings
+
+          negotiation = Negotiation.new(settings, agent_settings)
+          transport_v7 = Datadog::Core::Transport::HTTP.v7(**transport_options.dup)
+
+          @barrier = Barrier.new(BARRIER_TIMEOUT)
+
+          @client = Client.new(transport_v7, capabilities)
+          healthy = false
+          Datadog.logger.debug { "new remote configuration client: #{@client.id}" }
+
+          @worker = Worker.new(interval: settings.remote.poll_interval_seconds) do
+            unless healthy || negotiation.endpoint?('/v0.7/config')
+              @barrier.lift
+
+              next
+            end
+
+            begin
+              @client.sync
+              healthy ||= true
+            rescue Client::SyncError => e
+              Datadog.logger.error do
+                "remote worker client sync error: #{e.message} location: #{Array(e.backtrace).first}. skipping sync"
+              end
+            rescue StandardError => e
+              Datadog.logger.error do
+                "remote worker error: #{e.class.name} #{e.message} location: #{Array(e.backtrace).first}. "\
+                'reseting client state'
+              end
+
+              # client state is unknown, state might be corrupted
+              @client = Client.new(transport_v7, capabilities)
+              healthy = false
+              Datadog.logger.debug { "new remote configuration client: #{@client.id}" }
+
+              # TODO: bail out if too many errors?
+            end
+
+            @barrier.lift
+          end
+        end
+
+        def barrier(kind)
+          @worker.start
+
+          case kind
+          when :once
+            @barrier.wait_once
+          end
+        end
+
+        def shutdown!
+          @worker.stop
+        end
+
+        # Barrier provides a mechanism to fence execution until a condition happens
+        class Barrier
+          def initialize(timeout = nil)
+            @once = false
+            @timeout = timeout
+
+            @mutex = Mutex.new
+            @condition = ConditionVariable.new
+          end
+
+          # Wait for first lift to happen, otherwise don't wait
+          def wait_once(timeout = nil)
+            # TTAS (Test and Test-And-Set) optimisation
+            # Since @once only ever goes from false to true, this is semantically valid
+            return if @once
+
+            begin
+              @mutex.lock
+
+              return if @once
+
+              timeout ||= @timeout
+
+              # rbs/core has a bug, timeout type is incorrectly ?Integer
+              @condition.wait(@mutex, _ = timeout)
+            ensure
+              @mutex.unlock
+            end
+          end
+
+          # Wait for next lift to happen
+          def wait_next(timeout = nil)
+            @mutex.lock
+
+            timeout ||= @timeout
+
+            # rbs/core has a bug, timeout type is incorrectly ?Integer
+            @condition.wait(@mutex, _ = timeout)
+          ensure
+            @mutex.unlock
+          end
+
+          # Release all current waiters
+          def lift
+            @mutex.lock
+
+            @once ||= true
+
+            @condition.broadcast
+          ensure
+            @mutex.unlock
+          end
+        end
+
+        class << self
+          def build(settings, agent_settings)
+            return unless settings.remote.enabled
+
+            capabilities = Client::Capabilities.new(settings)
+
+            return if capabilities.products.empty?
+
+            negotiation = Negotiation.new(settings, agent_settings)
+
+            unless negotiation.endpoint?('/v0.7/config')
+              Datadog.logger.error do
+                'endpoint unavailable: disabling remote configuration for this process.'
+              end
+
+              return
+            end
+
+            Datadog.logger.debug { 'agent reachable and reports remote configuration endpoint' }
+
+            new(settings, capabilities, agent_settings)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/remote/configuration/content.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require_relative 'path'
+require_relative 'digest'
+
+module Datadog
+  module Core
+    module Remote
+      class Configuration
+        # Content stores the information associated with a specific Configuration::Path
+        class Content
+          class << self
+            def parse(hash)
+              path = Path.parse(hash[:path])
+              data = hash[:content]
+
+              new(path: path, data: data)
+            end
+          end
+
+          attr_reader :path, :data, :hashes
+          attr_accessor :version
+
+          def initialize(path:, data:)
+            @path = path
+            @data = data
+            @hashes = {}
+            @version = 0
+          end
+
+          def hexdigest(type)
+            @hashes[type] || compute_and_store_hash(type)
+          end
+
+          def length
+            @length ||= @data.size
+          end
+
+          private
+
+          def compute_and_store_hash(type)
+            @hashes[type] = Digest.hexdigest(type, @data)
+          end
+
+          private_class_method :new
+        end
+
+        # ContentList stores a list of Conetnt instances
+        # It provides convinient methods for finding content base on Configuration::Path and Configuration::Target
+        class ContentList < Array
+          class << self
+            def parse(array)
+              new.concat(array.map { |c| Content.parse(c) })
+            end
+          end
+
+          def find_content(path, target)
+            find { |c| c.path.eql?(path) && target.check(c) }
+          end
+
+          def [](path)
+            find { |c| c.path.eql?(path) }
+          end
+
+          def []=(path, content)
+            map! { |c| c.path.eql?(path) ? content : c }
+          end
+
+          def delete(path)
+            idx = index { |e| e.path.eql?(path) }
+
+            return if idx.nil?
+
+            delete_at(idx)
+          end
+
+          def paths
+            map(&:path).uniq
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/remote/configuration/digest.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module Datadog
+  module Core
+    module Remote
+      class Configuration
+        # Represent a list of Configuration::Digest
+        class DigestList < Array
+          class << self
+            def parse(hash)
+              new.concat(hash.map { |type, hexdigest| Digest.new(type, hexdigest) })
+            end
+          end
+
+          def check(content)
+            map { |digest| digest.check(content) }.reduce(:&)
+          end
+        end
+
+        # Stores and validates different cryptographic hash functions
+        class Digest
+          class InvalidHashTypeError < StandardError; end
+          attr_reader :type, :hexdigest
+
+          DIGEST_CHUNK = 1024
+
+          class << self
+            def hexdigest(type, data)
+              d = case type
+                  when :sha256
+                    ::Digest::SHA256.new
+                  when :sha512
+                    ::Digest::SHA512.new
+                  else
+                    raise InvalidHashTypeError, type
+                  end
+
+              while (buf = data.read(DIGEST_CHUNK))
+                d.update(buf)
+              end
+
+              d.hexdigest
+            ensure
+              data.rewind
+            end
+          end
+
+          def initialize(type, hexdigest)
+            @type = type.to_sym
+            @hexdigest = hexdigest
+          end
+
+          def check(content)
+            content.hexdigest(@type) == hexdigest
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/remote/configuration/path.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Remote
+      class Configuration
+        # Path stores path information
+        class Path
+          class ParseError < StandardError; end
+
+          class << self
+            RE = %r{
+              ^
+              (?<source>
+                datadog/(?<org_id>\d+)
+                |
+                employee
+              )
+              /
+              (?<product>[^/]+)
+              /
+              (?<config_id>[^/]+)
+              /
+              (?<name>[^/]+)
+              $
+            }mx.freeze
+
+            def parse(path)
+              m = RE.match(path)
+
+              raise ParseError, "could not parse: #{path.inspect}" if m.nil?
+
+              org_id = m['org_id'] ? m['org_id'].to_i : nil
+
+              source = m['source']
+              raise ParseError, 'missing source value' unless source
+
+              source = source.delete("/#{org_id}") if org_id
+
+              product = m['product']
+              raise ParseError, 'missing product value' unless product
+
+              config_id = m['config_id']
+              raise ParseError, 'missing config_id value' unless config_id
+
+              name = m['name']
+              raise ParseError, 'missing name value' unless name
+
+              new(source: source, org_id: org_id, product: product, config_id: config_id, name: name)
+            end
+          end
+
+          attr_reader :source, :org_id, :product, :config_id, :name
+
+          def initialize(source:, product:, config_id:, name:, org_id: nil)
+            @source = source
+            @org_id = org_id
+            @product = product
+            @config_id = config_id
+            @name = name
+          end
+
+          private_class_method :new
+
+          def to_s
+            if org_id
+              "#{source}/#{org_id}/#{product}/#{config_id}/#{name}"
+            else
+              "#{source}/#{product}/#{config_id}/#{name}"
+            end
+          end
+
+          def ==(other)
+            return false unless other.is_a?(Path)
+
+            to_s == other.to_s
+          end
+
+          def hash
+            to_s.hash
+          end
+
+          def eql?(other)
+            hash == other.hash
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/remote/configuration/repository.rb

@@ -0,0 +1,292 @@
+# frozen_string_literal: true
+
+require_relative 'content'
+
+module Datadog
+  module Core
+    module Remote
+      class Configuration
+        # Repository
+        class Repository
+          attr_reader \
+            :contents,
+            :opaque_backend_state,
+            :root_version,
+            :targets_version
+
+          UNVERIFIED_ROOT_VERSION = 1
+
+          INITIAL_TARGETS_VERSION = 0
+
+          def initialize
+            @contents = ContentList.new
+            @opaque_backend_state = nil
+            @root_version = UNVERIFIED_ROOT_VERSION
+            @targets_version = INITIAL_TARGETS_VERSION
+          end
+
+          def paths
+            @contents.paths
+          end
+
+          def [](path)
+            @contents[path]
+          end
+
+          def transaction
+            transaction = Transaction.new
+
+            yield(self, transaction)
+
+            commit(transaction)
+          end
+
+          def commit(transaction)
+            previous = contents.dup
+
+            touched = transaction.operations.each_with_object([]) do |op, acc|
+              acc << op.apply(self)
+            end
+
+            changes = ChangeSet.new
+
+            touched.uniq.each do |path|
+              next if path.nil?
+
+              changes.add(path, previous[path], @contents[path])
+            end
+
+            changes.freeze
+          end
+
+          def state
+            State.new(self)
+          end
+
+          # State store the repository state
+          class State
+            attr_reader \
+              :root_version,
+              :targets_version,
+              :config_states,
+              :has_error,
+              :error,
+              :opaque_backend_state,
+              :cached_target_files
+
+            def initialize(repository)
+              @repository = repository
+              @root_version = repository.root_version
+              @targets_version = repository.targets_version
+              @config_states = contents_to_config_states(repository.contents)
+              @has_error = false
+              @error = ''
+              @opaque_backend_state = repository.opaque_backend_state
+              @cached_target_files = contents_to_cached_target_files(repository.contents)
+            end
+
+            private
+
+            def contents_to_config_states(contents)
+              return [] if contents.empty?
+
+              contents.map do |content|
+                {
+                  id: content.path.config_id,
+                  version: content.version,
+                  product: content.path.product
+                }
+              end
+            end
+
+            def contents_to_cached_target_files(contents)
+              return [] if contents.empty?
+
+              contents.map do |content|
+                {
+                  path: content.path.to_s,
+                  length: content.length,
+                  hashes: content.hashes.map do |algorithm, hexdigest|
+                    {
+                      algorithm: algorithm,
+                      hash: hexdigest
+                    }
+                  end
+                }
+              end
+            end
+          end
+
+          # Encapsulates transaction operations
+          class Transaction
+            attr_reader :operations
+
+            def initialize
+              @operations = []
+            end
+
+            def delete(path)
+              @operations << Operation::Delete.new(path)
+            end
+
+            def insert(path, target, content)
+              @operations << Operation::Insert.new(path, target, content)
+            end
+
+            def update(path, target, content)
+              @operations << Operation::Update.new(path, target, content)
+            end
+
+            def set(**options)
+              @operations << Operation::Set.new(**options)
+            end
+          end
+
+          # Operation
+          module Operation
+            # Delete contents base on path
+            class Delete
+              attr_reader :path
+
+              def initialize(path)
+                super()
+                @path = path
+              end
+
+              def apply(repository)
+                return if repository[@path].nil?
+
+                repository.contents.delete(@path)
+
+                @path
+              end
+            end
+
+            # Insert content into the reporistory contents
+            class Insert
+              attr_reader :path, :target, :content
+
+              def initialize(path, target, content)
+                super()
+                @path = path
+                @target = target
+                @content = content
+              end
+
+              def apply(repository)
+                return unless repository[@path].nil?
+
+                @content.version = @target.version
+                repository.contents << @content
+
+                @path
+              end
+            end
+
+            # Update existimng repository's contents
+            class Update
+              attr_reader :path, :target, :content
+
+              def initialize(path, target, content)
+                super()
+                @path = path
+                @target = target
+                @content = content
+              end
+
+              def apply(repository)
+                return if repository[@path].nil?
+
+                @content.version = @target.version
+                repository.contents[@path] = @content
+
+                @path
+              end
+            end
+
+            # Set repository metadata
+            class Set
+              attr_reader :opaque_backend_state, :targets_version
+
+              def initialize(**options)
+                super()
+                @opaque_backend_state = options[:opaque_backend_state]
+                @targets_version = options[:targets_version]
+              end
+
+              def apply(repository)
+                repository.instance_variable_set(:@opaque_backend_state, @opaque_backend_state) if @opaque_backend_state
+
+                repository.instance_variable_set(:@targets_version, @targets_version) if @targets_version
+
+                nil
+              end
+            end
+          end
+
+          private_constant :Operation
+
+          module Change
+            # Delete change
+            class Deleted
+              attr_reader :path, :previous
+
+              def initialize(path, previous)
+                @path = path
+                @previous = previous
+              end
+            end
+
+            # Insert change
+            class Inserted
+              attr_reader :path, :content
+
+              def initialize(path, content)
+                @path = path
+                @content = content
+              end
+            end
+
+            # Update change
+            class Updated
+              attr_reader :path, :content, :previous
+
+              def initialize(path, content, previous)
+                @path = path
+                @content = content
+                @previous = previous
+              end
+            end
+          end
+
+          # Store list of Changes
+          class ChangeSet < Array
+            def paths
+              map(&:path)
+            end
+
+            def add(path, previous, content)
+              return if previous.nil? && content.nil?
+
+              return deleted(path, previous) if previous && content.nil?
+              return inserted(path, content) if content && previous.nil?
+              return updated(path, content, previous) if content && previous
+            end
+
+            def deleted(path, previous)
+              self << Change::Deleted.new(path, previous).freeze
+            end
+
+            def inserted(path, content)
+              self << Change::Inserted.new(path, content).freeze
+            end
+
+            def updated(path, content, previous)
+              self << Change::Updated.new(path, content, previous).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end