cw-datadog 2.23.0.2

data/lib/datadog/core/telemetry/worker.rb

@@ -0,0 +1,276 @@
+# frozen_string_literal: true
+
+require_relative 'event'
+
+require_relative '../utils/only_once_successful'
+require_relative '../workers/polling'
+require_relative '../workers/queue'
+
+module Datadog
+  module Core
+    module Telemetry
+      # Accumulates events and sends them to the API at a regular interval, including heartbeat event.
+      class Worker
+        include Core::Workers::Queue
+        include Core::Workers::Polling
+
+        DEFAULT_BUFFER_MAX_SIZE = 1000
+        APP_STARTED_EVENT_RETRIES = 10
+
+        def initialize(
+          heartbeat_interval_seconds:,
+          metrics_aggregation_interval_seconds:,
+          emitter:,
+          metrics_manager:,
+          dependency_collection:,
+          logger:,
+          enabled: true,
+          shutdown_timeout: Workers::Polling::DEFAULT_SHUTDOWN_TIMEOUT,
+          buffer_size: DEFAULT_BUFFER_MAX_SIZE
+        )
+          @emitter = emitter
+          @metrics_manager = metrics_manager
+          @dependency_collection = dependency_collection
+          @logger = logger
+
+          @ticks_per_heartbeat = (heartbeat_interval_seconds / metrics_aggregation_interval_seconds).to_i
+          @current_ticks = 0
+
+          # Workers::Polling settings
+          self.enabled = enabled
+          # Workers::IntervalLoop settings
+          self.loop_base_interval = metrics_aggregation_interval_seconds
+          self.fork_policy = Core::Workers::Async::Thread::FORK_POLICY_STOP
+
+          @shutdown_timeout = shutdown_timeout
+          @buffer_size = buffer_size
+
+          self.buffer = buffer_klass.new(@buffer_size)
+
+          @initial_event_once = Utils::OnlyOnceSuccessful.new(APP_STARTED_EVENT_RETRIES)
+        end
+
+        attr_reader :logger
+        attr_reader :initial_event_once
+        attr_reader :initial_event
+
+        # Returns true if worker thread is successfully started,
+        # false if worker thread was not started but telemetry is enabled,
+        # nil if telemetry is disabled.
+        def start(initial_event)
+          return if !enabled? || forked?
+
+          @initial_event = initial_event
+
+          # starts async worker
+          # perform should return true if thread was actually started,
+          # false otherwise
+          perform
+        end
+
+        def stop(force_stop = false, timeout = @shutdown_timeout)
+          buffer.close if running?
+
+          super
+        end
+
+        # Returns true if event was enqueued, nil if not.
+        # While returning false may seem more reasonable, the only reason
+        # for not enqueueing event (presently) is that telemetry is disabled
+        # altogether, and in this case other methods return nil.
+        def enqueue(event)
+          return if !enabled? || forked?
+
+          buffer.push(event)
+          true
+        end
+
+        def sent_initial_event?
+          initial_event_once.success?
+        end
+
+        def failed_initial_event?
+          initial_event_once.failed?
+        end
+
+        def need_initial_event?
+          !sent_initial_event? && !failed_initial_event?
+        end
+
+        # Wait for the worker to send out all events that have already
+        # been queued, up to 15 seconds. Returns whether all events have
+        # been flushed.
+        #
+        # @api private
+        def flush
+          return true unless enabled? || !run_loop?
+
+          started = Utils::Time.get_time
+          loop do
+            # The AppStarted event is triggered by the worker itself,
+            # from the worker thread. As such the main thread has no way
+            # to delay itself until that event is queued and we need some
+            # way to wait until that event is sent out to assert on it in
+            # the test suite. Check the run once flag which *should*
+            # indicate the event has been queued (at which point our queue
+            # depth check should waint until it's sent).
+            # This is still a hack because the flag can be overridden
+            # either way with or without the event being sent out.
+            # Note that if the AppStarted sending fails, this check
+            # will return false and flushing will be blocked until the
+            # 15 second timeout.
+            # Note that the first wait interval between telemetry event
+            # sending is 10 seconds, the timeout needs to be strictly
+            # greater than that.
+            return true if buffer.empty? && !in_iteration? && sent_initial_event?
+
+            sleep 0.5
+
+            return false if Utils::Time.get_time - started > 15
+          end
+        end
+
+        private
+
+        def perform(*events)
+          return if !enabled? || forked?
+
+          if need_initial_event?
+            started!
+            unless sent_initial_event?
+              # We still haven't succeeded in sending the started event,
+              # which will make flush_events do nothing - but the events
+              # given to us as the parameter have already been removed
+              # from the queue.
+              # Put the events back to the front of the queue to not
+              # lose them.
+              buffer.unshift(*events)
+              return
+            end
+          end
+
+          metric_events = @metrics_manager.flush!
+          events = [] if events.nil?
+          events += metric_events
+          if events.any?
+            flush_events(events)
+          end
+
+          @current_ticks += 1
+          return if @current_ticks < @ticks_per_heartbeat
+
+          @current_ticks = 0
+          heartbeat!
+        end
+
+        def flush_events(events)
+          events = deduplicate_logs(events)
+
+          logger.debug { "Sending #{events&.count} telemetry events" }
+          send_event(Event::MessageBatch.new(events))
+        end
+
+        def heartbeat!
+          return if !enabled? || !sent_initial_event?
+
+          send_event(Event::AppHeartbeat.new)
+        end
+
+        def started!
+          return unless enabled?
+
+          initial_event_once.run do
+            res = send_event(initial_event)
+
+            if res.ok?
+              logger.debug { "Telemetry initial event (#{initial_event.type}) is successfully sent" }
+
+              # TODO Dependencies loaded event should probably check for new
+              # dependencies and send the new ones.
+              # System tests demand only one instance of this event per
+              # dependency.
+              send_event(Event::AppDependenciesLoaded.new) if @dependency_collection && initial_event.class.eql?(Telemetry::Event::AppStarted) # standard:disable Style/ClassEqualityComparison:
+
+              true
+            else
+              logger.debug("Error sending telemetry initial event (#{initial_event.type}), retry after heartbeat interval...")
+              false
+            end
+          end
+
+          if failed_initial_event?
+            logger.debug { "Telemetry initial event (#{initial_event.type}) exhausted retries, disabling telemetry worker" }
+            disable!
+          end
+        end
+
+        def send_event(event)
+          res = @emitter.request(event)
+
+          disable_on_not_found!(res)
+
+          res
+        end
+
+        def dequeue
+          buffer.pop
+        end
+
+        def work_pending?
+          run_loop? || !buffer.empty?
+        end
+
+        def buffer_klass
+          if Core::Environment::Ext::RUBY_ENGINE == 'ruby'
+            Core::Buffer::CRuby
+          else
+            Core::Buffer::ThreadSafe
+          end
+        end
+
+        def disable!
+          self.enabled = false
+          @metrics_manager.disable!
+        end
+
+        def disable_on_not_found!(response)
+          return unless response.not_found?
+
+          logger.debug('Agent does not support telemetry; disabling future telemetry events.')
+          disable!
+        end
+
+        # Deduplicate logs by counting the number of repeated occurrences of the same log
+        # entry and replacing them with a single entry with the calculated `count` value.
+        # Non-log events are unchanged.
+        def deduplicate_logs(events)
+          return events if events.empty?
+
+          all_logs = []
+          other_events = events.reject do |event|
+            if event.is_a?(Event::Log)
+              all_logs << event
+              true
+            else
+              false
+            end
+          end
+
+          return events if all_logs.empty?
+
+          uniq_logs = all_logs.group_by(&:itself).map do |_, logs|
+            log = logs.first
+            if logs.size > 1
+              # New log event with a count of repeated occurrences
+              Event::Log.new(message: log.message, level: log.level, stack_trace: log.stack_trace, count: logs.size)
+            else
+              log
+            end
+          end
+
+          other_events + uniq_logs
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/transport/ext.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require_relative '../configuration/ext'
+module Datadog
+  module Core
+    module Transport
+      # @public_api
+      module Ext
+        module HTTP
+          HEADER_CONTAINER_ID = 'Datadog-Container-ID'
+          HEADER_DD_API_KEY = 'DD-API-KEY'
+          # Tells agent that `_dd.top_level` metrics have been set by the tracer.
+          # The agent will not calculate top-level spans but instead trust the tracer tagging.
+          #
+          # This prevents partially flushed traces being mistakenly marked as top-level.
+          #
+          # Setting this header to any non-empty value enables this feature.
+          HEADER_CLIENT_COMPUTED_TOP_LEVEL = 'Datadog-Client-Computed-Top-Level'
+          HEADER_CLIENT_COMPUTED_STATS = 'Datadog-Client-Computed-Stats'
+          HEADER_META_LANG = 'Datadog-Meta-Lang'
+          HEADER_META_LANG_VERSION = 'Datadog-Meta-Lang-Version'
+          HEADER_META_LANG_INTERPRETER = 'Datadog-Meta-Lang-Interpreter'
+          # Use for distinguishing between CRuby, JRuby, and TruffleRuby.
+          HEADER_META_LANG_INTERPRETER_VENDOR = 'Datadog-Meta-Lang-Interpreter-Vendor'
+          HEADER_META_TRACER_VERSION = 'Datadog-Meta-Tracer-Version'
+
+          # Header that prevents the Net::HTTP integration from tracing internal trace requests.
+          # Set it to any value to skip tracing.
+          HEADER_DD_INTERNAL_UNTRACED_REQUEST = 'DD-Internal-Untraced-Request'
+        end
+
+        module Test
+          ADAPTER = :test
+        end
+
+        module UnixSocket
+          ADAPTER = Datadog::Core::Configuration::Ext::Agent::UnixSocket::ADAPTER
+          DEFAULT_PATH = Datadog::Core::Configuration::Ext::Agent::UnixSocket::DEFAULT_PATH
+          DEFAULT_TIMEOUT_SECONDS = Datadog::Core::Configuration::Ext::Agent::UnixSocket::DEFAULT_TIMEOUT_SECONDS
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/transport/http/adapters/net.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+require_relative '../../../../core/transport/response'
+require_relative '../../../../core/vendor/multipart-post/net/http/post/multipart'
+
+module Datadog
+  module Core
+    module Transport
+      module HTTP
+        module Adapters
+          # Adapter for Net::HTTP
+          class Net
+            attr_reader \
+              :hostname,
+              :port,
+              :timeout,
+              :ssl
+
+            def initialize(agent_settings)
+              @hostname = agent_settings.hostname
+              @port = agent_settings.port
+              @timeout = agent_settings.timeout_seconds
+              @ssl = agent_settings.ssl
+            end
+
+            def self.build(agent_settings)
+              new(agent_settings)
+            end
+
+            def open(&block)
+              # DEV Initializing +Net::HTTP+ directly help us avoid expensive
+              # options processing done in +Net::HTTP.start+:
+              # debug cloudwise upload request
+              # https://github.com/ruby/ruby/blob/b2d96abb42abbe2e01f010ffc9ac51f0f9a50002/lib/net/http.rb#L614-L618
+              req = ::Net::HTTP.new(hostname, port, nil)
+
+              req.use_ssl = ssl
+              req.open_timeout = req.read_timeout = timeout
+
+              req.start(&block)
+            end
+
+            def call(env)
+              if respond_to?(env.verb)
+                send(env.verb, env)
+              else
+                raise UnknownHTTPMethod, env
+              end
+            end
+
+            def get(env)
+              get = ::Net::HTTP::Get.new(net_http_path_from_env(env), env.headers)
+
+              # Connect and send the request
+              http_response = open do |http|
+                http.request(get)
+              end
+
+              # Build and return response
+              Response.new(http_response)
+            end
+
+            def post(env)
+              post = nil
+
+              if env.form.nil? || env.form.empty?
+                post = ::Net::HTTP::Post.new(net_http_path_from_env(env), env.headers)
+                post.body = env.body
+              else
+                post = ::Datadog::Core::Vendor::Net::HTTP::Post::Multipart.new(
+                  env.path,
+                  env.form,
+                  env.headers
+                )
+              end
+
+              # Connect and send the request
+              http_response = open do |http|
+                http.request(post)
+              end
+
+              # Build and return response
+              Response.new(http_response)
+            end
+
+            def url
+              "http://#{hostname}:#{port}?timeout=#{timeout}"
+            end
+
+            def net_http_path_from_env(env)
+              path = env.path
+              case query = env.query
+              when Hash
+                path = path + '?' + URI.encode_www_form(query)
+              when String
+                path = path + '?' + query
+              when nil
+                # Nothing
+              else
+                raise ArgumentError, "Invalid type for query: #{query}"
+              end
+              path
+            end
+
+            # Raised when called with an unknown HTTP method
+            class UnknownHTTPMethod < StandardError
+              attr_reader :verb
+
+              def initialize(verb)
+                super("No matching Net::HTTP function for '#{verb}'!")
+              end
+            end
+
+            # A wrapped Net::HTTP response that implements the Transport::Response interface
+            class Response
+              include Datadog::Core::Transport::Response
+
+              attr_reader :http_response
+
+              def initialize(http_response)
+                @http_response = http_response
+              end
+
+              def payload
+                return super if http_response.nil?
+
+                http_response.body
+              end
+
+              def code
+                return super if http_response.nil?
+
+                http_response.code.to_i
+              end
+
+              def ok?
+                return super if http_response.nil?
+
+                code.between?(200, 299)
+              end
+
+              def unsupported?
+                return super if http_response.nil?
+
+                code == 415
+              end
+
+              def not_found?
+                return super if http_response.nil?
+
+                code == 404
+              end
+
+              def client_error?
+                return super if http_response.nil?
+
+                code.between?(400, 499)
+              end
+
+              def server_error?
+                return super if http_response.nil?
+
+                code.between?(500, 599)
+              end
+
+              def inspect
+                "#{super}, http_response:#{http_response}"
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/transport/http/adapters/registry.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Transport
+      module HTTP
+        module Adapters
+          # List of available adapters
+          class Registry
+            def initialize
+              @adapters = {}
+            end
+
+            def get(name)
+              @adapters[name]
+            end
+
+            def set(klass, name = nil)
+              name ||= klass.to_s
+              return if name.nil?
+
+              @adapters[name] = klass
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/transport/http/adapters/test.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require_relative '../../../../core/transport/response'
+
+module Datadog
+  module Core
+    module Transport
+      module HTTP
+        module Adapters
+          # Adapter for testing
+          class Test
+            attr_reader \
+              :buffer,
+              :status
+
+            # @param buffer [Array] an optional array that will capture all spans sent to this adapter, defaults to +nil+
+            # @deprecated Positional parameters are deprecated. Use named parameters instead.
+            def initialize(buffer = nil, **options)
+              @buffer = buffer || options[:buffer]
+              @mutex = Mutex.new
+              @status = 200
+            end
+
+            def call(env)
+              add_request(env)
+              Response.new(status)
+            end
+
+            def buffer?
+              !@buffer.nil?
+            end
+
+            def add_request(env)
+              @mutex.synchronize { buffer << env } if buffer?
+            end
+
+            def set_status!(status)
+              @status = status
+            end
+
+            def url
+            end
+
+            # Response for test adapter
+            class Response
+              include Datadog::Core::Transport::Response
+
+              attr_reader \
+                :body,
+                :code
+
+              def initialize(code, body = nil)
+                @code = code
+                @body = body
+              end
+
+              def payload
+                @body
+              end
+
+              def ok?
+                code.between?(200, 299)
+              end
+
+              def unsupported?
+                code == 415
+              end
+
+              def not_found?
+                code == 404
+              end
+
+              def client_error?
+                code.between?(400, 499)
+              end
+
+              def server_error?
+                code.between?(500, 599)
+              end
+
+              def inspect
+                "#{super}, code:#{code}"
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/transport/http/adapters/unix_socket.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require_relative '../../../transport/ext'
+require_relative 'net'
+
+module Datadog
+  module Core
+    module Transport
+      module HTTP
+        module Adapters
+          # Adapter for Unix sockets
+          class UnixSocket < Adapters::Net
+            attr_reader \
+              :filepath, # DEV(1.0): Rename to `uds_path`
+              :timeout
+
+            alias_method :uds_path, :filepath
+
+            # @deprecated Positional parameters are deprecated. Use named parameters instead.
+            # rubocop:disable Lint/MissingSuper
+            def initialize(uds_path = nil, **options)
+              @filepath = uds_path || options.fetch(:uds_path)
+              @timeout = options[:timeout] || Datadog::Core::Transport::Ext::UnixSocket::DEFAULT_TIMEOUT_SECONDS
+            end
+            # rubocop:enable Lint/MissingSuper
+
+            def self.build(agent_settings)
+              new(
+                uds_path: agent_settings.uds_path,
+                timeout: agent_settings.timeout_seconds,
+              )
+            end
+
+            def open(&block)
+              # Open connection
+              connection = HTTP.new(
+                uds_path,
+                read_timeout: timeout,
+                continue_timeout: timeout
+              )
+
+              connection.start(&block)
+            end
+
+            def url
+              "http+unix://#{uds_path}?timeout=#{timeout}"
+            end
+
+            # Re-implements Net:HTTP with underlying Unix socket
+            class HTTP < ::Net::HTTP
+              DEFAULT_TIMEOUT = 30
+
+              attr_reader \
+                :filepath, # DEV(1.0): Rename to `uds_path`
+                :unix_socket
+
+              alias_method :uds_path, :filepath
+
+              def initialize(uds_path, options = {})
+                super('localhost', 80)
+                @filepath = uds_path
+                @read_timeout = options.fetch(:read_timeout, DEFAULT_TIMEOUT)
+                @continue_timeout = options.fetch(:continue_timeout, DEFAULT_TIMEOUT)
+                @debug_output = options[:debug_output] if options.key?(:debug_output)
+              end
+
+              def connect
+                @unix_socket = UNIXSocket.open(uds_path)
+                @socket = ::Net::BufferedIO.new(@unix_socket).tap do |socket|
+                  socket.read_timeout = @read_timeout
+                  socket.continue_timeout = @continue_timeout
+                  socket.debug_output = @debug_output
+                end
+                on_connect
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end