instrumental_agent 2.0.0.alpha → 3.0.0.beta2

data/lib/instrumental/capistrano.rb

@@ -1,47 +1,5 @@
-require 'capistrano'
-require 'instrumental_agent'
-require 'etc'
-
-if Capistrano::Configuration.instance
-  Capistrano::Configuration.instance.load do
-    namespace :instrumental do
-      namespace :util do
-        desc "marker for beginning of deploy"
-        task :deploy_start do
-          set :instrumental_deploy_start, Time.now
-        end
-
-        desc "marker for end of deploy"
-        task :deploy_end do
-          set :instrumental_deploy_end, Time.now
-        end
-      end
-
-      desc "send a notice to instrumental about the deploy"
-      task :record_deploy_notice do
-        start_at = exists?(:instrumental_deploy_start) ? instrumental_deploy_start : Time.now
-        end_at = exists?(:instrumental_deploy_end) ? instrumental_deploy_end : start_at
-        deploy_duration_in_seconds = end_at - start_at
-        deployer = Etc.getlogin.chomp
-        agent_options = { :synchronous => true }
-        agent_options[:collector] = instrumental_host if exists?(:instrumental_host)
-        agent = Instrumental::Agent.new(instrumental_key, agent_options)
-        message = if exists?(:deploy_message)
-          deploy_message
-        else
-          "#{deployer} deployed #{current_revision}"
-        end
-        agent.notice(message,
-                     start_at,
-                     deploy_duration_in_seconds)
-        logger.info("Notified Instrumental of deployment")
-      end
-    end
-
-    before "deploy", "instrumental:util:deploy_start"
-    after  "deploy", "instrumental:util:deploy_end"
-    before "deploy:migrations", "instrumental:util:deploy_start"
-    after  "deploy:migrations", "instrumental:util:deploy_end"
-    after  "instrumental:util:deploy_end", "instrumental:record_deploy_notice"
-  end
+if Gem::Specification.find_by_name("capistrano").version >= Gem::Version.new("3.0.0")
+  load File.expand_path("../capistrano/capistrano3.rake", __FILE__)
+else
+  require_relative "capistrano/capistrano2"
 end

data/lib/instrumental/capistrano/capistrano2.rb

@@ -0,0 +1,47 @@
+require "etc"
+require "instrumental_agent"
+
+Capistrano::Configuration.instance.load do
+  _cset(:instrumental_hooks) { true }
+  _cset(:instrumental_key) { nil }
+  _cset(:deployer) { Etc.getlogin.chomp }
+
+  if fetch(:instrumental_hooks)
+    before "deploy", "instrumental:util:deploy_start"
+    after  "deploy", "instrumental:util:deploy_end"
+    before "deploy:migrations", "instrumental:util:deploy_start"
+    after  "deploy:migrations", "instrumental:util:deploy_end"
+    after  "instrumental:util:deploy_end", "instrumental:record_deploy_notice"
+  end
+
+  namespace :instrumental do
+    namespace :util do
+      desc "marker for beginning of deploy"
+      task :deploy_start do
+        set :instrumental_deploy_start, Time.now
+      end
+
+      desc "marker for end of deploy"
+      task :deploy_end do
+        set :instrumental_deploy_end, Time.now
+      end
+    end
+
+    desc "send a notice to instrumental about the deploy"
+    task :record_deploy_notice do
+      start_at                   = fetch(:instrumental_deploy_start, Time.now)
+      end_at                     = fetch(:instrumental_deploy_end, start_at)
+      deploy_duration_in_seconds = end_at - start_at
+      deployer                   = fetch(:deployer)
+      agent_options              = { :synchronous => true }
+      agent_options[:collector]  = instrumental_host if fetch(:instrumental_host, false)
+      agent                      = Instrumental::Agent.new(fetch(:instrumental_key), agent_options)
+      message                    = fetch(:deploy_message, "#{deployer} deployed #{current_revision}")
+
+      agent.notice(message,
+                   start_at,
+                   deploy_duration_in_seconds)
+      logger.info("Notified Instrumental of deployment")
+    end
+  end
+end

data/lib/instrumental/capistrano/capistrano3.rake

@@ -0,0 +1,56 @@
+require "etc"
+require "instrumental_agent"
+
+namespace :load do
+  task :defaults do
+    set :instrumental_hooks, true
+    set :instrumental_key,   nil
+    set :deployer,           Etc.getlogin.chomp
+  end
+end
+
+namespace :deploy do
+  before :starting, :check_instrumental_hooks do
+    invoke "instrumental:util:add_hooks" if fetch(:instrumental_hooks)
+  end
+end
+
+namespace :instrumental do
+  namespace :util do
+    desc "add instrumental hooks to deploy"
+    task :add_hooks do
+      before "deploy", "instrumental:util:deploy_start"
+      after  "deploy", "instrumental:util:deploy_end"
+      after  "instrumental:util:deploy_end", "instrumental:record_deploy_notice"
+    end
+
+    desc "marker for beginning of deploy"
+    task :deploy_start do
+      set :instrumental_deploy_start, Time.now
+    end
+
+    desc "marker for end of deploy"
+    task :deploy_end do
+      set :instrumental_deploy_end, Time.now
+    end
+  end
+
+  desc "send a notice to instrumental about the deploy"
+  task :record_deploy_notice do
+    start_at                   = fetch(:instrumental_deploy_start, Time.now)
+    end_at                     = fetch(:instrumental_deploy_end, start_at)
+    deploy_duration_in_seconds = end_at - start_at
+    deployer                   = fetch(:deployer)
+    agent_options              = { :synchronous => true }
+    agent_options[:collector]  = instrumental_host if fetch(:instrumental_host, false)
+    message                    = fetch(:deploy_message, "#{deployer} deployed #{fetch(:current_revision)}".strip)
+
+    if fetch(:instrumental_key)
+      agent = Instrumental::Agent.new(fetch(:instrumental_key), agent_options)
+      agent.notice(message,
+                   start_at,
+                   deploy_duration_in_seconds)
+      puts "Notified Instrumental of deployment"
+    end
+  end
+end

data/lib/instrumental/command_structs.rb

@@ -0,0 +1,32 @@
+module Instrumental
+  METRIC_TYPES = ["increment".freeze, "gauge".freeze].freeze
+
+  Command = Struct.new(:command, :metric, :value, :time, :count) do
+    def initialize(command, metric, value, time, count)
+      super(command, metric, value, time.to_i, count.to_i)
+    end
+
+    def to_s
+      [command, metric, value, time, count].map(&:to_s).join(" ")
+    end
+
+    def metadata
+      "#{metric}:#{time}".freeze
+    end
+
+    def +(other_command)
+      return self if other_command.nil?
+      Command.new(command, metric, value + other_command.value, time, count + other_command.count)
+    end
+  end
+
+  Notice = Struct.new(:note, :time, :duration) do
+    def initialize(note, time, duration)
+      super(note, time.to_i, duration.to_i)
+    end
+
+    def to_s
+      ["notice".freeze, time, duration, note].map(&:to_s).join(" ")
+    end
+  end
+end

data/lib/instrumental/event_aggregator.rb

@@ -0,0 +1,26 @@
+class EventAggregator
+  attr_accessor :counts, :values, :received_at, :frequency
+ 
+  def initialize(frequency:)
+    @values = Hash.new
+    @frequency = frequency
+  end
+
+  def put(command)
+    command_at = command.time
+    unless(command_at % frequency == 0)
+      command.time = (command_at - (command_at % frequency))
+    end
+    metadata = command.metadata
+    @values[metadata] = (command + @values[metadata])
+  end
+
+  def size
+    @values.size
+  end
+
+  def coerce_time(time)
+    itime = time.to_i
+    (itime - (itime % frequency)).to_i
+  end
+end

data/lib/instrumental/version.rb

@@ -1,3 +1,3 @@
 module Instrumental
-  VERSION = "2.0.0.alpha"
+  VERSION = "3.0.0.beta2"
 end

data/spec/agent_spec.rb

@@ -9,6 +9,8 @@ def wait(n=0.2, &block)
       if (Time.now - start) < 5
         sleep n
         retry
+      else
+        raise ex
       end
     end
   else
@@ -37,7 +39,8 @@ shared_examples "Instrumental Agent" do
     let(:token)        { 'test_token' }
     let(:address)      { server.host_and_port }
     let(:metrician)    { false }
-    let(:agent)        { Instrumental::Agent.new(token, :collector => address, :synchronous => synchronous, :enabled => enabled, :secure => secure?, :verify_cert => verify_cert?, :metrician => metrician) }
+    let(:frequency)    { 0 }
+    let(:agent)        { Instrumental::Agent.new(token, :collector => address, :synchronous => synchronous, :enabled => enabled, :secure => secure?, :verify_cert => verify_cert?, :metrician => metrician, :frequency => frequency) }
 
     # Server options
     let(:listen)       { true }
@@ -45,6 +48,12 @@ shared_examples "Instrumental Agent" do
     let(:authenticate) { true }
     let(:server)       { TestServer.new(:listen => listen, :authenticate => authenticate, :response => response, :secure => secure?) }
 
+    # Time Travel Options
+    let(:start_of_minute) do
+      now = Time.now.to_i
+      Time.at(now - (now % 60))
+    end
+
     before do
       Instrumental::Agent.logger.level = Logger::UNKNOWN
       @server = server
@@ -226,16 +235,16 @@ shared_examples "Instrumental Agent" do
             allow(agent.logger).to receive(:debug)
             expect(agent.logger).to receive(:debug).with("Dropping command, queue full(3): increment overflow_test 4 300 1")
             expect(agent.logger).to receive(:debug).with("Dropping command, queue full(3): increment overflow_test 5 300 1")
-            5.times do |i|
-              agent.increment('overflow_test', i + 1, 300)
-            end
-            wait do
-              expect(server.commands).to include("increment overflow_test 1 300 1")
-              expect(server.commands).to include("increment overflow_test 2 300 1")
-              expect(server.commands).to include("increment overflow_test 3 300 1")
-              expect(server.commands).to_not include("increment overflow_test 4 300 1")
-              expect(server.commands).to_not include("increment overflow_test 5 300 1")
+            1.upto(5) do |i|
+              agent.increment('overflow_test', i, 300)
             end
+
+            wait
+            expect(agent.sender_queue.size).to eq(3)
+            expect(agent.sender_queue.pop.first.to_s).to start_with("increment overflow_test 1 300 1")
+            expect(agent.sender_queue.pop.first.to_s).to start_with("increment overflow_test 2 300 1")
+            expect(agent.sender_queue.pop.first.to_s).to start_with("increment overflow_test 3 300 1")
+            expect(agent.sender_queue.size).to eq(0)
           end
         end
       end
@@ -246,7 +255,7 @@ shared_examples "Instrumental Agent" do
           5.times do |i|
             agent.increment('overflow_test', i + 1, 300)
           end
-          expect(agent.instance_variable_get(:@queue).size).to eq(0)
+          expect(agent.instance_variable_get(:@sender_queue).size).to eq(0)
           wait # let the server receive the commands
           expect(server.commands).to include("increment overflow_test 1 300 1")
           expect(server.commands).to include("increment overflow_test 2 300 1")
@@ -262,8 +271,10 @@ shared_examples "Instrumental Agent" do
           fork do
             agent.increment('fork_reconnect_test', 1, 3) # triggers reconnect
           end
+
           wait(1)
           agent.increment('fork_reconnect_test', 1, 4) # triggers reconnect
+
           wait(1)
           expect(server.connect_count).to eq(2)
 
@@ -279,17 +290,17 @@ shared_examples "Instrumental Agent" do
           sleep 1
         }
 
-        run_worker_loop_calls = 0
-        allow(agent).to receive(:run_worker_loop) {
-          run_worker_loop_calls += 1
+        run_sender_loop_calls = 0
+        allow(agent).to receive(:run_sender_loop) {
+          run_sender_loop_calls += 1
           sleep 3 # keep the worker thread alive
         }
 
         t = Thread.new { agent.increment("race") }
         agent.increment("race")
         wait(2)
-        expect(run_worker_loop_calls).to eq(1)
-        expect(agent.queue.size).to eq(2)
+        expect(run_sender_loop_calls).to eq(1)
+        expect(agent.sender_queue.size).to eq(2)
       end
 
       it "should never let an exception reach the user" do
@@ -312,14 +323,6 @@ shared_examples "Instrumental Agent" do
         expect(agent.increment("test")).to eq(nil)
       end
 
-      it "should track invalid metrics" do
-        expect(agent.logger).to receive(:warn).with(/%%/)
-        agent.increment(' %% .!#@$%^&*', 1, 1)
-        wait do
-          expect(server.commands.join("\n")).to include("increment agent.invalid_metric")
-        end
-      end
-
       it "should allow reasonable metric names" do
         agent.increment('a')
         agent.increment('a.b')
@@ -397,9 +400,9 @@ shared_examples "Instrumental Agent" do
 
       it "should allow flushing pending values to the server" do
         1.upto(100) { agent.gauge('a', rand(50)) }
-        expect(agent.instance_variable_get(:@queue).size).to be > 0
+        expect(agent.instance_variable_get(:@sender_queue).size).to be > 0
         agent.flush
-        expect(agent.instance_variable_get(:@queue).size).to eq(0)
+        expect(agent.instance_variable_get(:@sender_queue).size).to eq(0)
         wait do
           expect(server.commands.grep(/^gauge a /).size).to eq(100)
         end
@@ -437,7 +440,7 @@ shared_examples "Instrumental Agent" do
           agent.increment('reconnect_test', 1, 1234)
           wait
           # The agent should not have sent the metric yet, the server is not responding
-          expect(agent.queue.pop(true)).to include("increment reconnect_test 1 1234 1\n")
+          expect(agent.sender_queue.pop(true).first.to_s).to eq("increment reconnect_test 1 1234 1")
         end
 
         it "should warn once when buffer is full" do
@@ -472,14 +475,14 @@ shared_examples "Instrumental Agent" do
           agent.increment('reconnect_test', 1, 1234)
           wait
           # Since server hasn't responded to hello or authenticate, worker thread will not send data
-          expect(agent.queue.pop(true)).to include("increment reconnect_test 1 1234 1\n")
+          expect(agent.sender_queue.pop(true).first.to_s).to eq("increment reconnect_test 1 1234 1")
         end
       end
 
       context 'server hangup' do
         it "should cancel the worker thread when the host has hung up" do
           # Start the background agent thread and let it send one metric successfully
-          agent.gauge('connection_failure', 1, 1234)
+          agent.gauge('connection_failure1', 1, 1234)
           wait do
             expect(server.commands.grep(/connection_failure/).size).to eq(1)
           end
@@ -487,13 +490,13 @@ shared_examples "Instrumental Agent" do
           server.stop
           wait
           # Send one metric to the stopped server
-          agent.gauge('connection_failure', 1, 1234)
+          agent.gauge('connection_failure2', 1, 1234)
           # The agent thread should have stopped running since the network write would
           # have failed. The queue will still contain the metric that has yet to be sent
           wait do
             expect(agent.send(:running?)).to eq(false)
           end
-          expect(agent.queue.size).to eq(1)
+          expect(agent.sender_queue.size).to eq(1)
         end
 
         it "should restart the worker thread after hanging it up during an unreachable host event" do
@@ -512,7 +515,7 @@ shared_examples "Instrumental Agent" do
           wait do
             expect(agent.send(:running?)).to eq(false)
           end
-          expect(agent.queue.size).to eq(1)
+          expect(agent.sender_queue.size).to eq(1)
           # Start the server back up again
           server.listen
           # Sending another metric should kickstart the background worker thread
@@ -520,12 +523,76 @@ shared_examples "Instrumental Agent" do
           # The agent should now be running the background thread, and the queue should be empty
           wait do
             expect(agent.send(:running?)).to eq(true)
-            expect(agent.queue.size).to eq(0)
+            expect(agent.sender_queue.size).to eq(0)
           end
         end
 
-      end
+        it "should restart the worker thread after hanging it up during a bad ssl handshake event" do
+          # Start the background agent thread and let it send one metric successfully
+          agent.gauge('connection_failure', 1, 1234)
+          wait do
+            expect(server.commands.grep(/connection_failure/).size).to eq(1)
+          end
+          # Make the agent return the relevant exception on the next connection test
+          test_connection_fail = true
+          tc = agent.method(:test_connection)
+          allow(agent).to receive(:test_connection) do |*args, &block|
+            test_connection_fail ? raise(OpenSSL::SSL::SSLError.new) : tc.call(*args)
+          end
 
+          # Send one metric to the agent
+          agent.gauge('connection_failure', 1, 1234)
+          # The agent thread should have stopped running since the network write would
+          # have failed.
+          wait do
+            expect(agent.send(:running?)).to eq(false)
+          end
+          # The command is not in the queue
+          expect(agent.sender_queue.size).to eq(0)
+          # allow the agent to behave normally
+          test_connection_fail = false
+          # Sending another metric should kickstart the background worker thread
+          agent.gauge('connection_failure', 1, 1234)
+          # The agent should now be running the background thread, and the queue should be empty
+          wait do
+            expect(agent.send(:running?)).to eq(true)
+            expect(agent.sender_queue.size).to eq(0)
+            expect(server.commands.grep(/connection_failure/).size).to eq(2)
+          end
+        end
+
+        it "should accurately count failures so that backoff can work as intended" do
+          # Start the background agent thread and let it send one metric successfully
+          agent.gauge('connection_failure', 1, 1234)
+          wait do
+            expect(server.commands.grep(/connection_failure/).size).to eq(1)
+          end
+
+          # configure test_connection to fail in a way that won't kill the inner loop
+          test_connection_fail = true
+          tc = agent.method(:test_connection)
+          allow(agent).to receive(:test_connection) do |*args, &block|
+            test_connection_fail ? raise("test_connection_fail") : tc.call(*args)
+          end
+
+          # send some metrics
+          agent.gauge('connection_failure_1', 1, 1234)
+          agent.gauge('connection_failure_2', 1, 1234)
+          agent.gauge('connection_failure_3', 1, 1234)
+          wait do
+            expect(agent.instance_variable_get(:@failures)).to be > 0
+            expect(agent.sender_queue.size).to be > 0
+          end
+
+          # let the loop proceed
+          test_connection_fail = false
+
+          wait do
+            expect(agent.send(:running?)).to eq(true)
+            expect(agent.sender_queue.size).to eq(0)
+          end
+        end
+      end
 
       context 'not authenticating' do
         # Server will fail all authentication attempts
@@ -535,7 +602,7 @@ shared_examples "Instrumental Agent" do
           agent.increment('reconnect_test', 1, 1234)
           wait
           # Metrics should not have been sent since all authentication failed
-          expect(agent.queue.pop(true)).to include("increment reconnect_test 1 1234 1\n")
+          expect(agent.sender_queue.pop(true).first.to_s).to eq("increment reconnect_test 1 1234 1")
         end
       end
 
@@ -569,20 +636,21 @@ shared_examples "Instrumental Agent" do
           end
         end
 
-        it "should not wait to exit a process if there are no commands queued" do
+        it "should follow normal exit procedures whether or not there are commands queued" do
           allow(agent).to receive(:open_socket) { |*args, &block| sleep(5) && block.call }
-          with_constants('Instrumental::Agent::EXIT_FLUSH_TIMEOUT' => 3) do
-            if (pid = fork { agent.increment('foo', 1); agent.queue.clear })
+          with_constants('Instrumental::Agent::EXIT_FLUSH_TIMEOUT' => 1) do
+            if (pid = fork { agent.increment('foo', 1); agent.sender_queue.clear })
               tm = Time.now.to_f
               Process.wait(pid)
               diff = Time.now.to_f - tm
-              expect(diff).to be < 1
+              expect(diff).to be < 2
+              expect(diff).to be > 1
             end
           end
         end
       end
 
-      it "should not wait longer than EXIT_FLUSH_TIMEOUT to attempt flushing the socket when disconnecting" do
+      it "should not wait much longer than EXIT_FLUSH_TIMEOUT to attempt flushing the socket when disconnecting" do
         agent.increment('foo', 1)
         wait do
           expect(server.commands.grep(/foo/).size).to eq(1)
@@ -598,12 +666,13 @@ shared_examples "Instrumental Agent" do
               raise
             end
           end.join
-        end
+        end.at_least(1).times
+
         with_constants('Instrumental::Agent::EXIT_FLUSH_TIMEOUT' => 3) do
           tm = Time.now.to_f
           agent.cleanup
           diff = Time.now.to_f - tm
-          expect(diff).to be <= 3
+          expect(diff).to be <= 3.1
         end
       end
 
@@ -658,7 +727,7 @@ shared_examples "Instrumental Agent" do
           expect(agent.send(:running?)).to eq(true)
 
           # Setup a failure for the next command so we'll break out of the inner
-          # loop in run_worker_loop causing another call to open_socket
+          # loop in run_sender_loop causing another call to open_socket
           test_connection_fail = true
           tc = agent.method(:test_connection)
           allow(agent).to receive(:test_connection) { |*args, &block| test_connection_fail ? raise("fail") : tc.call(*args) }
@@ -735,6 +804,330 @@ shared_examples "Instrumental Agent" do
         end
       end
     end
+
+    describe Instrumental::Agent, "aggregation" do
+      context "aggregation enabled" do
+        let(:frequency) { 2 }
+
+        it "can be enabled at Agent.new time" do
+          expect(agent.frequency).to eq(2)
+        end
+
+        it "can be modified by setting the agent frequency" do
+          agent.frequency = 15
+          expect(agent.frequency).to eq(15)
+        end
+
+        it "is disabled by default" do
+          agent = Instrumental::Agent.new('test_token')
+          expect(agent.frequency.to_f).to eq(0)
+        end
+
+        it "should only allow frequencies that align with minutes" do
+          (-5..100).each do |freq|
+            agent.frequency = freq
+            expect(Instrumental::Agent::VALID_FREQUENCIES).to include(agent.frequency)
+          end
+        end
+
+        it "bypasses aggregator queue entirely for most commands when frequency == 0" do
+          agent.frequency = 0 # this is red - 0 for green
+          expect(EventAggregator).not_to receive(:new)
+          agent.increment('a_metric')
+        end
+
+        it "adds data to the event aggregator and does not immediately send it" do
+          Timecop.travel start_of_minute
+          agent.increment('test')
+          wait do
+            expect(agent.instance_variable_get(:@event_aggregator).size).to eq(1)
+            expect(agent.instance_variable_get(:@event_aggregator).values.values.first.metric).to eq('test')
+          end
+        end
+
+        it "batches data before sending" do
+          Timecop.freeze do
+            agent.increment('a_metric')
+            agent.increment('a_metric')
+            agent.increment('another_metric')
+          end
+          agent.flush(true)
+          wait do
+            expect(server.commands.grep(/_metric/).size).to eq(2)
+            aggregated_metric = server.commands.grep(/a_metric/).first.split(" ")
+            expect(aggregated_metric[2].to_i).to eq(2) # value
+            expect(aggregated_metric[4].to_i).to eq(2) # count
+          end
+        end
+
+        it "aggregates to the specified frequency within the aggregator" do
+          Timecop.travel(start_of_minute)
+          agent.frequency = 15
+          expect(agent.frequency).not_to be(Instrumental::Agent::DEFAULT_FREQUENCY)
+          agent.increment('metric', 1, Time.at(0))
+
+          # will get aligned to the closest frequency (15)
+          agent.increment('metric', 1, Time.at(20))
+          wait do
+            expect(agent.instance_variable_get(:@event_aggregator).values.keys).to eq(["metric:0", "metric:15"])
+          end
+          agent.flush
+          wait do
+            expect(server.commands.grep(/metric 1 0/).size).to eq(1)
+            expect(server.commands.grep(/metric 1 15/).size).to eq(1)
+          end
+        end
+
+        it "flushes data from both queues before sending" do
+          Timecop.freeze do
+            100.times do |i|
+              agent.increment("test_metric_#{i}")
+              agent.increment("other_metric")
+            end
+          end
+
+          expect(agent.instance_variable_get(:@aggregator_queue).size).to be > 0
+          agent.flush
+          expect(agent.instance_variable_get(:@sender_queue).size).to eq(0)
+          expect(agent.instance_variable_get(:@aggregator_queue).size).to eq(0)
+
+          wait do
+            expect(server.commands.grep(/test_metric/).size).to eq(100)
+            expect(server.commands.grep(/other_metric/).size).to eq(1)
+          end
+        end
+
+        it "does not batch notices" do
+          agent.frequency = 60
+          agent.notice "things are happening", 0, 100
+          agent.notice "things are happening", 0, 100
+          agent.notice "things are happening", 0, 100
+          wait do
+            expect(server.commands.grep(/things are happening/).size).to eq(3)
+          end
+        end
+
+        it "can be disabled by setting frequency to nil" do
+          agent.frequency = nil
+          expect(EventAggregator).not_to receive(:new)
+          agent.increment('metric')
+          wait do
+            expect(server.commands.grep(/metric/).size).to eq(1)
+          end
+        end
+
+        it "can be disabled by setting frequency to 0" do
+          agent.frequency = 0
+          expect(EventAggregator).not_to receive(:new)
+          agent.increment('metric')
+          wait do
+            expect(server.commands.grep(/metric/).size).to eq(1)
+          end
+        end
+
+        it "automatically uses the highest-without-going-over frequency for a bad frequency" do
+          agent.frequency = 17
+          expect(agent.frequency).to eq(15)
+          agent.frequency = 69420
+          expect(agent.frequency).to eq(60)
+          agent.frequency = 0
+          expect(agent.frequency).to eq(0)
+          agent.frequency = -1
+          expect(agent.frequency).to eq(0)
+        end
+
+        it "can take strings as frequency" do
+          agent = Instrumental::Agent.new('test_token', :frequency => "15")
+          expect(agent.frequency).to eq(15)
+        end
+
+        it "should not be enabled at the same time as synchronous" do
+          expect(Instrumental::Agent.logger).to receive(:warn).with(/Synchronous and Frequency should not be enabled at the same time! Defaulting to synchronous mode./)
+          agent = Instrumental::Agent.new('test_token', :synchronous => true, :frequency => 6)
+          expect(agent.synchronous).to eq(true)
+          expect(agent.frequency).to eq(0)
+        end
+
+        it "should use synchronous mode if it is enabled, even if turned on after frequency set at start" do
+          agent.increment('metric')
+          agent.increment('metric')
+          agent.synchronous = true
+          agent.increment('metric')
+          wait do
+            expect(server.commands.grep(/metric 1/).size).to eq(1)
+          end
+          agent.flush
+          wait do
+            expect(server.commands.grep(/metric 1/).size).to eq(1)
+            expect(server.commands.grep(/metric 2/).size).to eq(1)
+          end
+        end
+
+        it "sends aggregated metrics after specified frequency, even if no flush is sent" do
+          agent.frequency = 1
+          Timecop.travel(start_of_minute)
+          agent.increment('metric')
+          agent.increment('metric')
+          agent.gauge('other', 1)
+          agent.gauge('other', 1)
+          agent.gauge('other', 1)
+          sleep (0.5)
+          wait { expect(server.commands.grep(/metric/).size).to eq(0) }
+          sleep (0.51) # total sleep > 1 frequency
+
+          expect(server.commands.grep(/metric 2/).size).to eq(1)
+          expect(server.commands.grep(/other 3/).size).to eq(1)
+        end
+
+        # this test really relies on the worker threads not working unexpectedly
+        it "will overflow if the aggregator queue is full" do
+          Timecop.travel(start_of_minute)
+          with_constants('Instrumental::Agent::MAX_BUFFER' => 3) do
+            allow(agent.logger).to receive(:debug)
+            expect(agent.logger).to receive(:debug).with("Dropping command, queue full(3): increment overflow_test 4 300 1")
+            agent.increment('overflow_test', 4, 300, 1)
+            agent.increment('overflow_test', 4, 300, 1)
+            agent.increment('overflow_test', 4, 300, 1)
+            agent.increment('overflow_test', 4, 300, 1)
+
+            expect(agent.instance_variable_get(:@aggregator_queue).size).to eq(3)
+            agent.flush
+            expect(agent.instance_variable_get(:@aggregator_queue).size).to eq(0)
+          end
+        end
+
+        it "if aggregator is at max size, next command will force a forward to the sender thread" do
+          Timecop.travel(start_of_minute)
+          with_constants('Instrumental::Agent::MAX_AGGREGATOR_SIZE' => 3) do
+            agent.increment('overflow_test1')
+            agent.increment('overflow_test2')
+            agent.increment('overflow_test3')
+            agent.increment('overflow_test4')
+            agent.increment('overflow_test5')
+
+            # only 1 because the 5th command triggers a forward of the first 4
+            wait do
+              expect(agent.instance_variable_get(:@event_aggregator).size).to eq(1)
+            end
+            agent.flush
+            wait do
+              expect(server.commands.grep(/overflow_test/).size).to eq(5)
+            end
+          end
+        end
+
+        context do
+          let(:listen) { false }
+          it "will not send aggregators to the sender queue if the sender thread is not ready" do
+            Timecop.travel(start_of_minute)
+            agent.frequency = 1
+
+            with_constants('Instrumental::Agent::MAX_BUFFER' => 3,
+                          'Instrumental::Agent::MAX_AGGREGATOR_SIZE' => 4) do
+
+              # fill the queue
+              agent.increment('overflow_test1')
+              agent.increment('overflow_test2')
+              agent.increment('overflow_test3')
+
+              # wait until they are all in the aggregator
+              wait do
+                expect(agent.instance_variable_get(:@aggregator_queue).size).to eq(0)
+                expect(agent.instance_variable_get(:@event_aggregator).size).to eq(3)
+                expect(agent.instance_variable_get(:@sender_queue).size).to eq(0)
+              end
+
+              # fill the queue again
+              agent.increment('overflow_test1')
+              agent.increment('overflow_test2')
+              agent.increment('overflow_test3')
+
+              # wait until they are all in the aggregator
+              wait do
+                expect(agent.instance_variable_get(:@aggregator_queue).size).to eq(0)
+                expect(agent.instance_variable_get(:@event_aggregator).size).to eq(3)
+                expect(agent.instance_variable_get(:@sender_queue).size).to eq(0)
+              end
+
+              # wait for the aggregator to get forwarded and popped by the sender
+              wait do
+                expect(agent.instance_variable_get(:@aggregator_queue).size).to eq(0)
+                expect(agent.instance_variable_get(:@event_aggregator)).to eq(nil)
+                expect(agent.instance_variable_get(:@sender_queue).size).to eq(1)
+              end
+
+              # fill the queue again
+              agent.increment('overflow_test4')
+              agent.increment('overflow_test5')
+              agent.increment('overflow_test6')
+
+              # wait for them all to be in the aggregator
+              wait do
+                expect(agent.instance_variable_get(:@aggregator_queue).size).to eq(0)
+                expect(agent.instance_variable_get(:@event_aggregator).size).to eq(3)
+                expect(agent.instance_variable_get(:@sender_queue).size).to eq(1)
+              end
+
+              # sleep until the next forward is done
+              sleep(agent.frequency + 0.1)
+
+              # fill the queue again
+              agent.increment('overflow_test7')
+              agent.increment('overflow_test8')
+              agent.increment('overflow_test9')
+
+              # because sending is blocked, the prevous aggregator never sent
+              # when it hits max size, the aggregator queue starts backing up
+              wait do
+                expect(agent.instance_variable_get(:@aggregator_queue).size).to eq(1)
+                expect(agent.instance_variable_get(:@event_aggregator).size).to eq(5)
+                expect(agent.instance_variable_get(:@sender_queue).size).to eq(1)
+              end
+
+              # send 3 more items, to overflow the aggregator queue
+              allow(agent.logger).to receive(:debug)
+              expect(agent.logger).to receive(:debug).with("Dropping command, queue full(3): increment overflow_testc 4 300 1")
+              agent.increment('overflow_testa')
+              agent.increment('overflow_testb')
+              agent.increment('overflow_testc', 4, 300, 1) # will get dropped
+
+              wait do
+                expect(agent.instance_variable_get(:@aggregator_queue).size).to eq(3)
+                expect(agent.instance_variable_get(:@event_aggregator).size).to eq(5)
+                expect(agent.instance_variable_get(:@sender_queue).size).to eq(1)
+              end
+            end
+          end
+        end
+
+        if FORK_SUPPORTED
+          it "should automatically reconnect when forked when aggregation is enabled" do
+            Timecop.travel start_of_minute
+            agent.frequency = 10
+
+            agent.increment('fork_reconnect_test1', 1, 0, 1)
+            fork do
+              agent.increment('fork_reconnect_test2', 1, 0, 1) # triggers reconnect
+              exit
+            end
+
+
+            sleep 1
+            agent.increment('fork_reconnect_test3', 1, 0, 1) # triggers reconnect
+
+            agent.flush
+            expect(server.connect_count).to eq(2)
+
+            wait do
+              expect(server.commands).to include("increment fork_reconnect_test1 1 0 1")
+              expect(server.commands).to include("increment fork_reconnect_test2 1 0 1")
+              expect(server.commands).to include("increment fork_reconnect_test3 1 0 1")
+              expect(server.commands.grep(/fork_reconnect/).size).to eq(3)
+            end
+          end
+        end
+      end
+    end
   end
 end