instrumental_agent 2.0.0 → 3.0.0.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 7fec654f621fd1e8f11f79fca789690d2e3d9b52
-  data.tar.gz: 078bf7a001fab6ed9f982cf818e675def3aaf65f
+SHA256:
+  metadata.gz: 20bc1dfd989a1d555912b21cd1a43b0a2ebb235e411fed6b27746cd3c01ce2ad
+  data.tar.gz: 7e725c0d514db6cccba3827a76a7b73995b06084a54bf0637cf7c2423d2beb3b
 SHA512:
-  metadata.gz: 4d2b8bdfa044822f00fb9b682817d2bc1c9108e26b544b93ee218212d7ce3b41ff88c0c422de21300e35638b7490f5a5032096d469ab322d1ab3f0c369012c63
-  data.tar.gz: 8a3494eb5e9c5932d54a7a2192d28ac181963c5d4f98567ca5cf54aa3381582eb2bed88439df664dd88a6fd4c0689c37e5ec8489e82d0fc15fffd58702cc04a8
+  metadata.gz: 356538412bb4aeeb4af6dbb912de961ca587b08410a48cc60b8246e7d138909dab5f75261dd441bf4fd9287868565c2a4554a313c0b9d43316ca7ac615f38b20
+  data.tar.gz: f819635e2b8ea0efaa7a25ed62cdf52d54b602e8b8703a7eea3b7975f83ed557ffe0cdd1a39b5dd97942492e3a872d4e343c6c179de18feb3dbb257c08faf1a8

data/.ruby-version

@@ -1 +1 @@
-2.0.0-p648
+2.6.3

data/.travis.yml

@@ -1,8 +1,7 @@
 sudo: false
 language: ruby
 rvm:
-  - 2.0.0-p648
-  - 2.1.5
-  - 2.2.3
-  - 2.3.0
-  - 2.4.0
+  - 2.5.7
+  - 2.6.3
+  - 2.6.6
+  - 2.7.1

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+### 3.0.0.beta [October 8, 2020]
+* Drop support for outdated versions of Ruby
+* Explicitly add support for new versions of Ruby
+* Add support for client-side aggregation
+* Note: the agent API has NOT changed.  This is a major release because of the significant changes in Ruby versions officially supported.
+
+### 3.0.0.alpha [August 22, 2019]
+* Drop support for outdated versions of Ruby
+* Explicitly add support for new versions of Ruby
+* Better handling of SSL errors when connecting to Instrumental
+* Note: the agent API has NOT changed.  This is a major release because of the significant changes in Ruby versions officially supported.
+
+### 2.1.0 [January 19, 2018]
+* Add support for capistrano 3
+
 ### 2.0.0 [August 21, 2017]
 * Add automatic tracking of common application metrics, official release
 

data/Gemfile

@@ -1,11 +1,6 @@
 source "https://rubygems.org"
 
 gemspec
-ruby_engine = defined?(RUBY_ENGINE) && RUBY_ENGINE
-if RUBY_VERSION < "1.9" && !%w{jruby rbx}.include?(ruby_engine)
-  # Built and installed via ext/mkrf_conf.rb
-  gem 'system_timer', '~> 1.2'
-end
 
 # fixes 2.3.0 ffi bundle error
-gem 'ffi', '~> 1.0.11'
+gem 'ffi', '~> 1.0.11'

data/README.md

@@ -59,6 +59,17 @@ User.find_each do |user|
 end
 ```
 
+## Aggregation
+Aggregation collects more data on your system before sending it to Instrumental. This reduces the total amount of data being sent, at the cost of a small amount of additional latency. You can control this feature with the frequency parameter:
+
+```ruby
+I = Instrumental::Agent.new('PROJECT_API_TOKEN', :frequency => 15) # send data every 15 seconds
+I.frequency = 6 # send batches of data every 6 seconds
+```
+
+The agent may send data more frequently if you are sending a large number of different metrics. Values between 3 and 15 are generally reasonable. If you want to disable this behavior and send every metric as fast as possible, set frequency to zero or nil. Note that a frequency of zero will still use a seperate thread for performance - it is NOT the same as synchronous mode.
+
+
 ## Server Metrics
 
 Want server stats like load, memory, etc.? Check out [InstrumentalD](https://github.com/instrumental/instrumentald).
@@ -109,6 +120,17 @@ I = Instrumental::Agent.new('PROJECT_API_TOKEN',
 )
 ```
 
+### Upgrading from 2.x
+
+Agent version 3.x drops support for some older rubies, but should otherwise be a drop-in replacement. If you wish to enable Aggregation, enable the agent with the frequency option set to the number of seconds you would like to wait between flushes. For example:
+
+```
+I = Instrumental::Agent.new('PROJECT_API_TOKEN',
+  :enabled => Rails.env.production?,
+  :frequency => 15
+)
+```
+
 ## Troubleshooting & Help
 
 We are here to help. Email us at [support@instrumentalapp.com](mailto:support@instrumentalapp.com).

data/instrumental_agent.gemspec

@@ -4,13 +4,13 @@ require "instrumental/version"
 Gem::Specification.new do |s|
   s.name        = "instrumental_agent"
   s.version     = Instrumental::VERSION
-  s.authors     = ["Elijah Miller", "Christopher Zelenak", "Kristopher Chambers", "Matthew Hassfurder"]
+  s.authors     = ["Expected Behavior"]
   s.email       = ["support@instrumentalapp.com"]
   s.homepage    = "http://github.com/instrumental/instrumental_agent-ruby"
   s.summary     = %q{Custom metric monitoring for Ruby applications via Instrumental}
   s.description = %q{This agent supports Instrumental custom metric monitoring for Ruby applications. It provides high-data reliability at high scale, without ever blocking your process or causing an exception.}
   s.license     = "MIT"
-  s.required_ruby_version = '>= 2.0.0'
+  s.required_ruby_version = '>= 2.5.7'
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/instrumental/agent.rb

@@ -1,5 +1,7 @@
 require 'instrumental/version'
 require 'instrumental/system_timer'
+require 'instrumental/command_structs'
+require 'instrumental/event_aggregator'
 require 'logger'
 require 'openssl' rescue nil
 require 'resolv'
@@ -15,14 +17,17 @@ module Instrumental
     EXIT_FLUSH_TIMEOUT                 = 5
     HOSTNAME                           = Socket.gethostbyname(Socket.gethostname).first rescue Socket.gethostname
     MAX_BUFFER                         = 5000
+    MAX_AGGREGATOR_SIZE                = 5000
     MAX_RECONNECT_DELAY                = 15
     REPLY_TIMEOUT                      = 10
     RESOLUTION_FAILURES_BEFORE_WAITING = 3
     RESOLUTION_WAIT                    = 30
     RESOLVE_TIMEOUT                    = 1
+    DEFAULT_FREQUENCY                  = 0
+    VALID_FREQUENCIES                  = [0, 1, 2, 3, 4, 5, 6, 10, 12, 15, 20, 30, 60]
 
 
-    attr_accessor :host, :port, :synchronous, :queue, :dns_resolutions, :last_connect_at
+    attr_accessor :host, :port, :synchronous, :frequency, :sender_queue, :aggregator_queue, :dns_resolutions, :last_connect_at
     attr_reader :connection, :enabled, :secure
 
     def self.logger=(l)
@@ -52,6 +57,7 @@ module Instrumental
       # port:        8001
       # enabled:     true
       # synchronous: false
+      # frequency:   10
       # secure:      true
       # verify:      true
       @api_key         = api_key
@@ -73,14 +79,23 @@ module Instrumental
       @port            = (@port || default_port).to_i
       @enabled         = options.has_key?(:enabled) ? !!options[:enabled] : true
       @synchronous     = !!options[:synchronous]
+
+      if options.has_key?(:frequency)
+        self.frequency = options[:frequency]
+      else
+        self.frequency = DEFAULT_FREQUENCY
+      end
+
+      @metrician       = options[:metrician].nil? ? true : !!options[:metrician]
       @pid             = Process.pid
       @allow_reconnect = true
-      @certs           = certificates
       @dns_resolutions = 0
       @last_connect_at = 0
-      @metrician       = options[:metrician].nil? ? true : !!options[:metrician]
+
       @start_worker_mutex = Mutex.new
-      @queue = Queue.new
+      @aggregator_queue = Queue.new
+      @sender_queue = Queue.new
+
 
       setup_cleanup_at_exit if @enabled
 
@@ -94,7 +109,9 @@ module Instrumental
     #  agent.gauge('load', 1.23)
     def gauge(metric, value, time = Time.now, count = 1)
       if valid?(metric, value, time, count) &&
-          send_command("gauge", metric, value, time.to_i, count.to_i)
+         send_command(Instrumental::Command.new("gauge".freeze, metric, value, time, count))
+        # tempted to "gauge" this to a symbol? Don't. Frozen strings are very fast,
+        # and later we're going to to_s every one of these anyway.
         value
       else
         nil
@@ -142,7 +159,7 @@ module Instrumental
     #  agent.increment('users')
     def increment(metric, value = 1, time = Time.now, count = 1)
       if valid?(metric, value, time, count) &&
-          send_command("increment", metric, value, time.to_i, count.to_i)
+         send_command(Instrumental::Command.new("increment".freeze, metric, value, time, count))
         value
       else
         nil
@@ -157,7 +174,7 @@ module Instrumental
     #  agent.notice('A notice')
     def notice(note, time = Time.now, duration = 0)
       if valid_note?(note)
-        send_command("notice", time.to_i, duration.to_i, note)
+        send_command(Instrumental::Notice.new(note, time, duration))
         note
       else
         nil
@@ -196,6 +213,22 @@ module Instrumental
       @logger || self.class.logger
     end
 
+    def frequency=(frequency)
+      freq = frequency.to_i
+      if !VALID_FREQUENCIES.include?(freq)
+        logger.warn "Frequency must be a value that divides evenly into 60: 1, 2, 3, 4, 5, 6, 10, 12, 15, 20, 30, or 60."
+        # this will make all negative numbers and nils into 0s
+        freq = VALID_FREQUENCIES.select{ |f| f < freq }.max.to_i
+      end
+
+      @frequency = if(@synchronous)
+                     logger.warn "Synchronous and Frequency should not be enabled at the same time! Defaulting to synchronous mode."
+                     0
+                   else
+                     freq
+                   end
+    end
+
     # Stopping the agent will immediately stop all communication
     # to Instrumental.  If you call this and submit another metric,
     # the agent will start again.
@@ -207,12 +240,19 @@ module Instrumental
     #
     def stop
       disconnect
-      if @thread
-        @thread.kill
-        @thread = nil
+      if @sender_thread
+        @sender_thread.kill
+        @sender_thread = nil
+      end
+      if @aggregator_thread
+        @aggregator_thread.kill
+        @aggregator_thread = nil
       end
-      if @queue
-        @queue.clear
+      if @sender_queue
+        @sender_queue.clear
+      end
+      if @aggregator_queue
+        @aggregator_queue.clear
       end
     end
 
@@ -222,18 +262,25 @@ module Instrumental
     # where at_exit is bypassed like Resque workers.
     def cleanup
       if running?
-        logger.info "Cleaning up agent, queue size: #{@queue.size}, thread running: #{@thread.alive?}"
+        logger.info "Cleaning up agent, aggregator_size: #{@aggregator_queue.size}, thread_running: #{@aggregator_thread.alive?}"
+        logger.info "Cleaning up agent, queue size: #{@sender_queue.size}, thread running: #{@sender_thread.alive?}"
         @allow_reconnect = false
-        if @queue.size > 0
-          queue_message('exit')
-          begin
-            with_timeout(EXIT_FLUSH_TIMEOUT) { @thread.join }
-          rescue Timeout::Error
-            if @queue.size > 0
-              logger.error "Timed out working agent thread on exit, dropping #{@queue.size} metrics"
-            else
-              logger.error "Timed out Instrumental Agent, exiting"
-            end
+        begin
+          with_timeout(EXIT_FLUSH_TIMEOUT) do
+            @aggregator_queue << ['exit']
+            @aggregator_thread.join
+            @sender_queue << ['exit']
+            @sender_thread.join
+          end
+        rescue Timeout::Error
+          total_size = @sender_queue&.size.to_i +
+                       @aggregator_queue&.size.to_i +
+                       @event_aggregator&.size.to_i
+
+          if total_size > 0
+            logger.error "Timed out working agent thread on exit, dropping #{total_size} metrics"
+          else
+            logger.error "Timed out Instrumental Agent, exiting"
           end
         end
       end
@@ -271,7 +318,8 @@ module Instrumental
     end
 
     def report_exception(e)
-      logger.error "Exception occurred: #{e.message}\n#{e.backtrace.join("\n")}"
+      # puts "--- Exception of type #{e.class} occurred:\n#{e.message}\n#{e.backtrace.join("\n")}"
+      logger.error "Exception of type #{e.class} occurred:\n#{e.message}\n#{e.backtrace.join("\n")}"
     end
 
     def ipv4_address_for_host(host, port, moment_to_connect = Time.now.to_i)
@@ -291,44 +339,41 @@ module Instrumental
       nil
     end
 
-    def send_command(cmd, *args)
-      cmd = "%s %s\n" % [cmd, args.collect { |a| a.to_s }.join(" ")]
-      if enabled?
-
-        start_connection_worker
-        if @queue && @queue.size < MAX_BUFFER
-          @queue_full_warning = false
-          logger.debug "Queueing: #{cmd.chomp}"
-          queue_message(cmd, { :synchronous => @synchronous })
-        else
-          if !@queue_full_warning
-            @queue_full_warning = true
-            logger.warn "Queue full(#{@queue.size}), dropping commands..."
-          end
-          logger.debug "Dropping command, queue full(#{@queue.size}): #{cmd.chomp}"
-          nil
-        end
+    def send_command(command)
+      return logger.debug(command.to_s) unless enabled?
+      start_workers
+      critical_queue = frequency.to_i == 0 ? @sender_queue : @aggregator_queue
+      if critical_queue && critical_queue.size < MAX_BUFFER
+        @queue_full_warning = false
+        logger.debug "Queueing: #{command.to_s}"
+        queue_message(command, { :synchronous => @synchronous })
       else
-        logger.debug cmd.strip
+        if !@queue_full_warning
+          @queue_full_warning = true
+          logger.warn "Queue full(#{critical_queue.size}), dropping commands..."
+        end
+        logger.debug "Dropping command, queue full(#{critical_queue.size}): #{command.to_s}"
+        nil
       end
     end
 
     def queue_message(message, options = {})
-      if @enabled
-        options ||= {}
-        if options[:allow_reconnect].nil?
-          options[:allow_reconnect] = @allow_reconnect
-        end
-        synchronous = options.delete(:synchronous)
-        if synchronous
-          options[:sync_resource] ||= ConditionVariable.new
-          @sync_mutex.synchronize {
-            @queue << [message, options]
-            options[:sync_resource].wait(@sync_mutex)
-          }
-        else
-          @queue << [message, options]
-        end
+      return message unless enabled?
+
+      # imagine it's a reverse merge, but with fewer allocations
+      options[:allow_reconnect] = @allow_reconnect unless options.has_key?(:allow_reconnect)
+
+      if options.delete(:synchronous)
+        options[:sync_resource] ||= ConditionVariable.new
+        @sync_mutex.synchronize {
+          queue = message == "flush" ? @aggregator_queue : @sender_queue
+          queue << [message, options]
+          options[:sync_resource].wait(@sync_mutex)
+        }
+      elsif frequency.to_i == 0
+        @sender_queue << [message, options]
+      else
+        @aggregator_queue << [message, options]
       end
       message
     end
@@ -350,31 +395,15 @@ module Instrumental
 
     def test_connection
       begin
-        # In the case where the socket is an OpenSSL::SSL::SSLSocket,
-        # on Ruby 1.8.6, 1.8.7 or 1.9.1, read_nonblock does not exist,
-        # and so the case of testing socket liveliness via a nonblocking
-        # read that catches a wait condition won't work.
-        #
-        # We grab the SSL socket's underlying IO object and perform the
-        # non blocking read there in order to ensure the socket is still
-        # valid
-        if @socket.respond_to?(:read_nonblock)
-          @socket.read_nonblock(1)
-        elsif @socket.respond_to?(:io)
-          # The SSL Socket may send down additional data at close time,
-          # so we perform two nonblocking reads, one to pull any pending
-          # data on the socket, and the second to actually perform the connection
-          # liveliness test
-          @socket.io.read_nonblock(1024) && @socket.io.read_nonblock(1024)
-        end
+        @socket.read_nonblock(1)
       rescue *wait_exceptions
         # noop
       end
     end
 
-    def start_connection_worker
+    def start_workers
       # NOTE: We need a mutex around both `running?` and thread creation,
-      # otherwise we could create two threads.
+      # otherwise we could create too many threads.
       # Return early and queue the message if another thread is
       # starting the worker.
       return if !@start_worker_mutex.try_lock
@@ -384,13 +413,34 @@ module Instrumental
         disconnect
         address = ipv4_address_for_host(@host, @port)
         if address
-          @pid = Process.pid
+          new_pid = if @pid != Process.pid
+                      @pid = Process.pid
+                      true
+                    else
+                      false
+                    end
+
           @sync_mutex = Mutex.new
           @failures = 0
           @sockaddr_in = Socket.pack_sockaddr_in(@port, address)
-          logger.info "Starting thread"
-          @thread = Thread.new do
-            run_worker_loop
+
+          logger.info "Starting aggregator thread"
+          if !@aggregator_thread&.alive?
+            if new_pid
+              @event_aggregator = nil
+              @aggregator_queue = Queue.new
+            end
+            @aggregator_thread = Thread.new do
+              run_aggregator_loop
+            end
+          end
+
+          if !@sender_thread&.alive?
+            logger.info "Starting sender thread"
+            @sender_queue = Queue.new if new_pid
+            @sender_thread = Thread.new do
+              run_sender_loop
+            end
           end
         end
       ensure
@@ -426,82 +476,155 @@ module Instrumental
       sock
     end
 
-    def run_worker_loop
-      command_and_args = nil
-      command_options = nil
-      logger.info "connecting to collector"
-      with_timeout(CONNECT_TIMEOUT) do
-        @socket = open_socket(@sockaddr_in, @secure, @verify_cert)
+    def run_aggregator_loop
+      # if the sender queue is some level of full, should we keep aggregating until it empties out?
+      # what does this mean for aggregation slices - aggregating to nearest frequency will
+      # make the object needlessly larger, when minute resolution is what we have on the server
+      begin
+        loop do
+          now = Time.now.to_i
+          time_to_wait = if frequency == 0
+                           0
+                         else
+                           next_frequency = (now - (now % frequency)) + frequency
+                           time_to_wait = [(next_frequency - Time.now.to_f), 0].max
+                         end
+
+          command_and_args, command_options = if @event_aggregator&.size.to_i > MAX_AGGREGATOR_SIZE
+                                                logger.info "Aggregator full, flushing early with #{MAX_AGGREGATOR_SIZE} metrics."
+                                                command_and_args, command_options = ['forward', {}]
+                                              else
+                                                begin
+                                                  with_timeout(time_to_wait) do
+                                                    @aggregator_queue.pop
+                                                  end
+                                                rescue Timeout::Error
+                                                  ['forward', {}]
+                                                end
+                                              end
+          if command_and_args
+            case command_and_args
+            when 'exit'
+              if !@event_aggregator.nil?
+                @sender_queue << @event_aggregator
+                @event_aggregator = nil
+              end
+              logger.info "Exiting, #{@aggregator_queue.size} commands remain"
+              return true
+            when 'flush'
+              if !@event_aggregator.nil?
+                @sender_queue << @event_aggregator
+                @event_aggregator = nil
+              end
+              @sender_queue << ['flush', command_options]
+            when 'forward'
+              if !@event_aggregator.nil?
+                next if @sender_queue.size > 0 && @sender_queue.num_waiting < 1
+                @sender_queue << @event_aggregator
+                @event_aggregator = nil
+              end
+            when Notice
+              @sender_queue << [command_and_args, command_options]
+            else
+              @event_aggregator = EventAggregator.new(frequency: @frequency) if @event_aggregator.nil?
+
+              logger.debug "Sending: #{command_and_args} to aggregator"
+              @event_aggregator.put(command_and_args)
+            end
+            command_and_args = nil
+            command_options = nil
+          end
+        end
+      rescue Exception => err
+        report_exception(err)
       end
-      logger.info "connected to collector at #{host}:#{port}"
-      hello_options = {
-        "version" => "ruby/instrumental_agent/#{VERSION}",
-        "hostname" => HOSTNAME,
-        "pid" => Process.pid,
-        "runtime" => "#{defined?(RUBY_ENGINE) ? RUBY_ENGINE : "ruby"}/#{RUBY_VERSION}p#{RUBY_PATCHLEVEL}",
-        "platform" => RUBY_PLATFORM
-      }.to_a.flatten.map { |v| v.to_s.gsub(/\s+/, "_") }.join(" ")
-
-      send_with_reply_timeout "hello #{hello_options}"
-      send_with_reply_timeout "authenticate #{@api_key}"
+    end
+
+    def run_sender_loop
       @failures = 0
-      loop do
-        command_and_args, command_options = @queue.pop
-        if command_and_args
-          sync_resource = command_options && command_options[:sync_resource]
-          test_connection
-          case command_and_args
-          when 'exit'
-            logger.info "Exiting, #{@queue.size} commands remain"
-            return true
-          when 'flush'
-            release_resource = true
-          else
-            logger.debug "Sending: #{command_and_args.chomp}"
-            @socket.puts command_and_args
-          end
-          command_and_args = nil
-          command_options = nil
-          if sync_resource
-            @sync_mutex.synchronize do
-              sync_resource.signal
+      begin
+        logger.info "connecting to collector"
+        command_and_args = nil
+        command_options = nil
+        with_timeout(CONNECT_TIMEOUT) do
+          @socket = open_socket(@sockaddr_in, @secure, @verify_cert)
+        end
+        logger.info "connected to collector at #{host}:#{port}"
+        hello_options = {
+          "version" => "ruby/instrumental_agent/#{VERSION}",
+          "hostname" => HOSTNAME,
+          "pid" => Process.pid,
+          "runtime" => "#{defined?(RUBY_ENGINE) ? RUBY_ENGINE : "ruby"}/#{RUBY_VERSION}p#{RUBY_PATCHLEVEL}",
+          "platform" => RUBY_PLATFORM
+        }.to_a.flatten.map { |v| v.to_s.gsub(/\s+/, "_") }.join(" ")
+
+        send_with_reply_timeout "hello #{hello_options}"
+        send_with_reply_timeout "authenticate #{@api_key}"
+
+        loop do
+          command_and_args, command_options = @sender_queue.pop
+          if command_and_args
+            sync_resource = command_options && command_options[:sync_resource]
+            test_connection
+            case command_and_args
+            when 'exit'
+              logger.info "Exiting, #{@sender_queue.size} commands remain"
+              return true
+            when 'flush'
+              release_resource = true
+            when EventAggregator
+              command_and_args.values.values.each do |command|
+                logger.debug "Sending: #{command}"
+                @socket.puts command
+              end
+            else
+              logger.debug "Sending: #{command_and_args}"
+              @socket.puts command_and_args
+            end
+            command_and_args = nil
+            command_options = nil
+            if sync_resource
+              @sync_mutex.synchronize do
+                sync_resource.signal
+              end
             end
           end
         end
-      end
-    rescue Exception => err
-      allow_reconnect = @allow_reconnect
-      case err
-      when EOFError
+      rescue Exception => err
+        allow_reconnect = @allow_reconnect
+        case err
+        when EOFError
         # nop
-      when Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::EADDRINUSE, Timeout::Error
-        # If the connection has been refused by Instrumental
-        # or we cannot reach the server
-        # or the connection state of this socket is in a race
-        logger.error "unable to connect to Instrumental, hanging up with #{@queue.size} messages remaining"
-        logger.debug "Exception: #{err.inspect}\n#{err.backtrace.join("\n")}"
-        allow_reconnect = false
-      else
-        report_exception(err)
-      end
-      if allow_reconnect == false ||
-        (command_options && command_options[:allow_reconnect] == false)
-        logger.info "Not trying to reconnect"
-        @failures = 0
-        return
-      end
-      if command_and_args
-        logger.debug "requeueing: #{command_and_args}"
-        @queue << command_and_args
+        when Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::EADDRINUSE, Timeout::Error, OpenSSL::SSL::SSLError
+          # If the connection has been refused by Instrumental
+          # or we cannot reach the server
+          # or the connection state of this socket is in a race
+          # or SSL is not functioning properly for some reason
+          logger.error "unable to connect to Instrumental, hanging up with #{@sender_queue.size} messages remaining"
+          logger.debug "Exception: #{err.inspect}\n#{err.backtrace.join("\n")}"
+          allow_reconnect = false
+        else
+          report_exception(err)
+        end
+        if allow_reconnect == false ||
+           (command_options && command_options[:allow_reconnect] == false)
+          logger.info "Not trying to reconnect"
+          @failures = 0
+          return
+        end
+        if command_and_args
+          logger.debug "requeueing: #{command_and_args}"
+          @sender_queue << command_and_args
+        end
+        disconnect
+        @failures += 1
+        delay = [(@failures - 1) ** BACKOFF, MAX_RECONNECT_DELAY].min
+        logger.error "disconnected, #{@failures} failures in a row, reconnect in #{delay}..."
+        sleep delay
+        retry
+      ensure
+        disconnect
       end
-      disconnect
-      @failures += 1
-      delay = [(@failures - 1) ** BACKOFF, MAX_RECONNECT_DELAY].min
-      logger.error "disconnected, #{@failures} failures in a row, reconnect in #{delay}..."
-      sleep delay
-      retry
-    ensure
-      disconnect
     end
 
     def setup_cleanup_at_exit
@@ -511,7 +634,11 @@ module Instrumental
     end
 
     def running?
-      !@thread.nil? && @pid == Process.pid && @thread.alive?
+      !@sender_thread.nil? &&
+        !@aggregator_thread.nil? &&
+        @pid == Process.pid &&
+        @sender_thread.alive? &&
+        @aggregator_thread.alive?
     end
 
     def flush_socket(socket)
@@ -541,18 +668,5 @@ module Instrumental
     def allows_secure?
       defined?(OpenSSL)
     end
-
-    def certificates
-      if allows_secure?
-        base_dir = File.expand_path(File.join(File.dirname(__FILE__), "..", ".."))
-        %w{equifax geotrust rapidssl}.map do |name|
-          OpenSSL::X509::Certificate.new(File.open(File.join(base_dir, "certs", "#{name}.ca.pem")))
-        end
-      else
-        []
-      end
-    end
-
   end
-
 end