elastic-apm 2.6.1 → 2.7.0

data/lib/elastic_apm/railtie.rb

@@ -23,7 +23,10 @@ module ElasticAPM
     end
 
     config.after_initialize do
-      require 'elastic_apm/spies/action_dispatch'
+      if ElasticAPM.running? &&
+         !ElasticAPM.agent.config.disabled_spies.include?('action_dispatch')
+        require 'elastic_apm/spies/action_dispatch'
+      end
     end
 
     private
@@ -31,8 +34,10 @@ module ElasticAPM
     # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
     def start(config)
       if (reason = should_skip?(config))
-        config.alert_logger.info "Skipping because: #{reason}. " \
-          "Start manually with `ElasticAPM.start'"
+        unless config.disable_start_message?
+          config.alert_logger.info "Skipping because: #{reason}. " \
+            "Start manually with `ElasticAPM.start'"
+        end
         return
       end
 

data/lib/elastic_apm/spies/faraday.rb

@@ -5,6 +5,14 @@ module ElasticAPM
   module Spies
     # @api private
     class FaradaySpy
+      def self.without_net_http
+        return yield unless defined?(NetHTTPSpy)
+
+        ElasticAPM::Spies::NetHTTPSpy.disable_in do
+          yield
+        end
+      end
+
       # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
       # rubocop:disable Metrics/BlockLength, Metrics/PerceivedComplexity
       # rubocop:disable Metrics/CyclomaticComplexity
@@ -32,7 +40,7 @@ module ElasticAPM
             type = "ext.faraday.#{method}"
 
             ElasticAPM.with_span name, type do |span|
-              ElasticAPM::Spies::NetHTTPSpy.disable_in do
+              ElasticAPM::Spies::FaradaySpy.without_net_http do
                 trace_context = span&.trace_context || transaction.trace_context
 
                 run_request_without_apm(method, url, body, headers) do |req|

data/lib/elastic_apm/transport/base.rb

@@ -5,40 +5,70 @@ require 'elastic_apm/transport/connection'
 require 'elastic_apm/transport/worker'
 require 'elastic_apm/transport/serializers'
 require 'elastic_apm/transport/filters'
+require 'elastic_apm/util/throttle'
 
 module ElasticAPM
   module Transport
+    # rubocop:disable Metrics/ClassLength
     # @api private
     class Base
       include Logging
 
+      WATCHER_EXECUTION_INTERVAL = 5
+      WATCHER_TIMEOUT_INTERVAL = 4
+      WORKER_JOIN_TIMEOUT = 5
+
       def initialize(config)
         @config = config
         @queue = SizedQueue.new(config.api_buffer_size)
-        @pool = Concurrent::FixedThreadPool.new(config.pool_size)
-        @workers = []
 
         @serializers = Serializers.new(config)
         @filters = Filters.new(config)
+
+        @stopped = Concurrent::AtomicBoolean.new
+        @workers = Array.new(config.pool_size)
+
+        @watcher_mutex = Mutex.new
+        @worker_mutex = Mutex.new
       end
 
-      attr_reader :config, :queue, :workers, :filters
+      attr_reader :config, :queue, :filters, :workers, :watcher, :stopped
 
       def start
+        debug '%s: Starting Transport', pid_str
+
+        ensure_watcher_running
+        ensure_worker_count
       end
 
       def stop
+        debug '%s: Stopping Transport', pid_str
+
+        @stopped.make_true
+
+        stop_watcher
         stop_workers
       end
 
+      # rubocop:disable Metrics/MethodLength
       def submit(resource)
+        if @stopped.true?
+          warn '%s: Transport stopping, no new events accepted', pid_str
+          return false
+        end
+
+        ensure_watcher_running
         queue.push(resource, true)
 
-        ensure_worker_count
+        true
       rescue ThreadError
-        warn 'Queue is full (%i items), skipping…', config.api_buffer_size
+        throttled_queue_full_warning
+        nil
+      rescue Exception => e
+        error '%s: Failed adding to the transport queue: %p', pid_str, e.inspect
         nil
       end
+      # rubocop:enable Metrics/MethodLength
 
       def add_filter(key, callback)
         @filters.add(key, callback)
@@ -46,52 +76,99 @@ module ElasticAPM
 
       private
 
+      def pid_str
+        format('[PID:%s]', Process.pid)
+      end
+
+      def ensure_watcher_running
+        # pid has changed == we've forked
+        return if @pid == Process.pid
+
+        @watcher_mutex.synchronize do
+          return if @pid == Process.pid
+          @pid = Process.pid
+
+          @watcher = Concurrent::TimerTask.execute(
+            execution_interval: WATCHER_EXECUTION_INTERVAL,
+            timeout_interval: WATCHER_TIMEOUT_INTERVAL
+          ) { ensure_worker_count }
+        end
+      end
+
       def ensure_worker_count
-        missing = config.pool_size - @workers.length
-        return unless missing > 0
+        @worker_mutex.synchronize do
+          return if all_workers_alive?
+          return if stopped.true?
 
-        info 'Booting %i workers', missing
-        missing.times { boot_worker }
+          @workers.map! do |thread|
+            next thread if thread&.alive?
+
+            boot_worker
+          end
+        end
+      end
+
+      def all_workers_alive?
+        !!workers.all? { |t| t&.alive? }
       end
 
-      # rubocop:disable Metrics/MethodLength
       def boot_worker
-        worker = Worker.new(
-          config,
-          queue,
-          serializers: @serializers,
-          filters: @filters
-        )
-
-        @workers.push worker
-
-        @pool.post do
-          worker.work_forever
-          @workers.delete(worker)
+        debug '%s: Booting worker...', pid_str
+
+        Thread.new do
+          Worker.new(
+            config, queue,
+            serializers: @serializers,
+            filters: @filters
+          ).work_forever
         end
       end
-      # rubocop:enable Metrics/MethodLength
 
+      # rubocop:disable Metrics/MethodLength
       def stop_workers
-        return unless @pool.running?
+        debug '%s: Stopping workers', pid_str
 
-        debug 'Stopping workers'
         send_stop_messages
 
-        debug 'Shutting down pool'
-        @pool.shutdown
+        @worker_mutex.synchronize do
+          workers.each do |thread|
+            next if thread.nil?
+            next if thread.join(WORKER_JOIN_TIMEOUT)
 
-        return if @pool.wait_for_termination(5)
+            debug(
+              '%s: Worker did not stop in %ds, killing...',
+              pid_str, WORKER_JOIN_TIMEOUT
+            )
+            thread.kill
+          end
 
-        warn "Worker pool didn't close in 5 secs, killing ..."
-        @pool.kill
+          @workers.clear
+        end
       end
+      # rubocop:enable Metrics/MethodLength
 
       def send_stop_messages
-        @workers.each { queue.push(Worker::StopMessage.new, true) }
+        config.pool_size.times { queue.push(Worker::StopMessage.new, true) }
       rescue ThreadError
         warn 'Cannot push stop messages to worker queue as it is full'
       end
+
+      def stop_watcher
+        @watcher_mutex.synchronize do
+          return if watcher.nil? || @pid != Process.pid
+          watcher.shutdown
+        end
+      end
+
+      def throttled_queue_full_warning
+        (@queue_full_log ||= Util::Throttle.new(5) do
+          warn(
+            '%s: Queue is full (%i items), skipping…',
+            pid_str, config.api_buffer_size
+          )
+        end).call
+      end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/elastic_apm/transport/connection.rb

@@ -1,204 +1,128 @@
 # frozen_string_literal: true
 
-require 'http'
 require 'concurrent'
 require 'zlib'
 
+require 'elastic_apm/transport/connection/http'
+
 module ElasticAPM
   module Transport
     # @api private
-    class Connection # rubocop:disable Metrics/ClassLength
+    class Connection
       include Logging
 
-      class FailedToConnectError < InternalError; end
-
-      # @api private
-      # HTTP.rb calls #rewind the body stream which IO.pipes don't support
-      class ModdedIO < IO
-        def self.pipe(ext_enc = nil)
-          super(ext_enc).tap do |rw|
-            rw[0].define_singleton_method(:rewind) { nil }
-          end
-        end
-      end
+      # A connection holds an instance `http` of an Http::Connection.
+      #
+      # The HTTP::Connection itself is not thread safe.
+      #
+      # The connection sends write requests and close requests to `http`, and
+      # has to ensure no write requests are sent after closing `http`.
+      #
+      # The connection schedules a separate thread to close an `http`
+      # connection some time in the future. To avoid the thread interfering
+      # with ongoing write requests to `http`, write and close
+      # requests have to be synchronized.
 
       HEADERS = {
         'Content-Type' => 'application/x-ndjson',
         'Transfer-Encoding' => 'chunked'
       }.freeze
-      GZIP_HEADERS = HEADERS.merge('Content-Encoding' => 'gzip').freeze
+      GZIP_HEADERS = HEADERS.merge(
+        'Content-Encoding' => 'gzip'
+      ).freeze
 
-      # rubocop:disable Metrics/MethodLength
       def initialize(config, metadata)
         @config = config
-        @metadata = metadata.to_json
-
+        @metadata = JSON.fast_generate(metadata)
         @url = config.server_url + '/intake/v2/events'
-
-        headers =
-          (@config.http_compression? ? GZIP_HEADERS : HEADERS).dup
-
-        if (token = config.secret_token)
-          headers['Authorization'] = "Bearer #{token}"
-        end
-
-        @client = HTTP.headers(headers).persistent(@url)
-
-        configure_proxy
-        configure_ssl
-
+        @headers = build_headers
+        @ssl_context = build_ssl_context
         @mutex = Mutex.new
       end
-      # rubocop:enable Metrics/MethodLength
-
-      def configure_proxy
-        unless @config.proxy_address && @config.proxy_port
-          return
-        end
-
-        @client = @client.via(
-          @config.proxy_address,
-          @config.proxy_port,
-          @config.proxy_username,
-          @config.proxy_password,
-          @config.proxy_headers
-        )
-      end
 
-      def configure_ssl
-        return unless @config.use_ssl? && @config.server_ca_cert
-
-        @ssl_context = OpenSSL::SSL::SSLContext.new.tap do |context|
-          context.ca_file = @config.server_ca_cert
-        end
-      end
+      attr_reader :http
 
+      # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
       def write(str)
-        return if @config.disable_send
-
-        connect_unless_connected
-
-        @mutex.synchronize { append(str) }
-
-        return unless @bytes_sent >= @config.api_request_size
-
-        flush
-      rescue FailedToConnectError => e
-        error "Couldn't establish connection to APM Server:\n%p", e
-        flush
-
-        nil
-      end
+        return false if @config.disable_send
 
-      def connected?
-        @mutex.synchronize { @connected }
-      end
+        begin
+          bytes_written = 0
 
-      def flush
-        @mutex.synchronize do
-          return unless @connected
+          # The request might get closed from timertask so let's make sure we
+          # hold it open until we've written.
+          @mutex.synchronize do
+            connect if http.nil? || http.closed?
+            bytes_written = http.write(str)
+          end
 
-          debug 'Closing request'
-          @wr.close
-          @conn_thread.join 5 if @conn_thread
+          flush(:api_request_size) if bytes_written >= @config.api_request_size
+        rescue IOError => e
+          error('Connection error: %s', e.inspect)
+          flush(:ioerror)
+        rescue Errno::EPIPE => e
+          error('Connection error: %s', e.inspect)
+          flush(:broken_pipe)
+        rescue Exception => e
+          error('Connection error: %s', e.inspect)
+          flush(:connection_error)
         end
       end
+      # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
 
-      private
-
-      # rubocop:disable Metrics/MethodLength
-      def connect_unless_connected
+      def flush(reason = :force)
+        # Could happen from the timertask so we need to sync
         @mutex.synchronize do
-          return true if @connected
-
-          debug 'Opening new request'
-
-          reset!
-
-          @rd, @wr = ModdedIO.pipe
-
-          enable_compression! if @config.http_compression?
-
-          perform_request_in_thread
-          wait_for_connection
-
-          schedule_closing if @config.api_request_time
-
-          append(@metadata)
-
-          true
+          return if http.nil?
+          http.close(reason)
         end
       end
-      # rubocop:enable Metrics/MethodLength
-
-      # rubocop:disable Metrics/MethodLength
-      def perform_request_in_thread
-        @conn_thread = Thread.new do
-          begin
-            @connected = true
-
-            resp = @client.post(
-              @url,
-              body: @rd,
-              ssl_context: @ssl_context
-            ).flush
-          rescue Exception => e
-            @connection_error = e
-          ensure
-            @connected = false
-          end
 
-          if resp&.status == 202
-            debug 'APM Server responded with status 202'
-          elsif resp
-            error "APM Server responded with an error:\n%p", resp.body.to_s
-          end
-
-          resp
-        end
+      def inspect
+        format(
+          '@%s http connection closed? :%s>',
+          super.split.first,
+          http.closed?
+        )
       end
-      # rubocop:enable Metrics/MethodLength
-
-      def append(str)
-        bytes =
-          if @config.http_compression
-            @bytes_sent = @wr.tell
-          else
-            @bytes_sent += str.bytesize
-          end
 
-        debug 'Bytes sent during this request: %d', bytes
+      private
+
+      def connect
+        schedule_closing if @config.api_request_time
 
-        @wr.puts(str)
+        @http =
+          Http.open(
+            @config, @url,
+            headers: @headers,
+            ssl_context: @ssl_context
+          ).tap { |http| http.write(@metadata) }
       end
+      # rubocop:enable
 
       def schedule_closing
+        @close_task&.cancel
         @close_task =
           Concurrent::ScheduledTask.execute(@config.api_request_time) do
-            flush
+            flush(:timeout)
           end
       end
 
-      def enable_compression!
-        @wr.binmode
-        @wr = Zlib::GzipWriter.new(@wr)
-      end
-
-      def reset!
-        @bytes_sent = 0
-        @connected = false
-        @connection_error = nil
-        @close_task = nil
+      def build_headers
+        (
+          @config.http_compression? ? GZIP_HEADERS : HEADERS
+        ).dup.tap do |headers|
+          if (token = @config.secret_token)
+            headers['Authorization'] = "Bearer #{token}"
+          end
+        end
       end
 
-      def wait_for_connection
-        until @connected
-          if (exception = @connection_error)
-            @wr&.close
-            raise FailedToConnectError, exception
-          end
+      def build_ssl_context
+        return unless @config.use_ssl? && @config.server_ca_cert
 
-          sleep 0.01
+        OpenSSL::SSL::SSLContext.new.tap do |context|
+          context.ca_file = @config.server_ca_cert
         end
       end
     end