instrumental_agent 3.0.0.alpha → 3.0.0.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a45dc47aabd6ade8ea7198ea34f4bbc205a4979b1cae1ed18dbfabb4109df0b8
-  data.tar.gz: 67ebcbf6fca85da2087e6196dbcf57752ebe3470411c54241487ce834c2f5cf7
+  metadata.gz: a1a781a0abec98e8c737a57fac5134fe2ee847879949fc5134ced2c0b0136076
+  data.tar.gz: 89950c6e0b59713dc6034be99acfc52473ef049b63824685694626d049a85b8b
 SHA512:
-  metadata.gz: 1bfe306e6511e588aa58a258510b18f79fe4b70180b7ee8bd4fea0b967d80cf8cab3765228b4cd385f99537704557f918d92972b9cf6da0b646225b4a5c6bd87
-  data.tar.gz: feaf73f66d1ed593936bcd3aecf4e0ebccbc67fcf104905e28ad67e9d4e14575a0a3259a5c1fc9b2ac21ef212c2be64910c757bfc931668b48799d3466eeacc8
+  metadata.gz: ca82be6040189400c0e01513d517202e8d27d900ad2f10e5558025cad6a499e49985ee58c95efdf90d24e2fd0174b9349c7cd6ac8f906325f2de766c0f14cba3
+  data.tar.gz: 742200da53f676a9341a795f5f8145416650c033466782348fc4852dc62e656e9fed4e2be02c0829c2ec946092e7dfcd7d836cc50e731a6ceb8cb2493b76a1b6

data/.travis.yml

@@ -1,6 +1,7 @@
 sudo: false
 language: ruby
 rvm:
-  - 2.4.6
-  - 2.5.5
+  - 2.5.7
   - 2.6.3
+  - 2.6.6
+  - 2.7.1

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+### 3.0.0.beta [October 8, 2020]
+* Drop support for outdated versions of Ruby
+* Explicitly add support for new versions of Ruby
+* Add support for client-side aggregation
+* Note: the agent API has NOT changed.  This is a major release because of the significant changes in Ruby versions officially supported.
+
 ### 3.0.0.alpha [August 22, 2019]
 * Drop support for outdated versions of Ruby
 * Explicitly add support for new versions of Ruby

data/README.md

@@ -59,6 +59,17 @@ User.find_each do |user|
 end
 ```
 
+## Aggregation
+Aggregation collects more data on your system before sending it to Instrumental. This reduces the total amount of data being sent, at the cost of a small amount of additional latency. You can control this feature with the frequency parameter:
+
+```ruby
+I = Instrumental::Agent.new('PROJECT_API_TOKEN', :frequency => 15) # send data every 15 seconds
+I.frequency = 6 # send batches of data every 6 seconds
+```
+
+The agent may send data more frequently if you are sending a large number of different metrics. Values between 3 and 15 are generally reasonable. If you want to disable this behavior and send every metric as fast as possible, set frequency to zero or nil. Note that a frequency of zero will still use a seperate thread for performance - it is NOT the same as synchronous mode.
+
+
 ## Server Metrics
 
 Want server stats like load, memory, etc.? Check out [InstrumentalD](https://github.com/instrumental/instrumentald).
@@ -109,6 +120,17 @@ I = Instrumental::Agent.new('PROJECT_API_TOKEN',
 )
 ```
 
+### Upgrading from 2.x
+
+Agent version 3.x drops support for some older rubies, but should otherwise be a drop-in replacement. If you wish to enable Aggregation, enable the agent with the frequency option set to the number of seconds you would like to wait between flushes. For example:
+
+```
+I = Instrumental::Agent.new('PROJECT_API_TOKEN',
+  :enabled => Rails.env.production?,
+  :frequency => 15
+)
+```
+
 ## Troubleshooting & Help
 
 We are here to help. Email us at [support@instrumentalapp.com](mailto:support@instrumentalapp.com).

data/instrumental_agent.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
   s.summary     = %q{Custom metric monitoring for Ruby applications via Instrumental}
   s.description = %q{This agent supports Instrumental custom metric monitoring for Ruby applications. It provides high-data reliability at high scale, without ever blocking your process or causing an exception.}
   s.license     = "MIT"
-  s.required_ruby_version = '>= 2.4.6'
+  s.required_ruby_version = '>= 2.5.7'
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/instrumental/agent.rb

@@ -1,5 +1,7 @@
 require 'instrumental/version'
 require 'instrumental/system_timer'
+require 'instrumental/command_structs'
+require 'instrumental/event_aggregator'
 require 'logger'
 require 'openssl' rescue nil
 require 'resolv'
@@ -15,14 +17,17 @@ module Instrumental
     EXIT_FLUSH_TIMEOUT                 = 5
     HOSTNAME                           = Socket.gethostbyname(Socket.gethostname).first rescue Socket.gethostname
     MAX_BUFFER                         = 5000
+    MAX_AGGREGATOR_SIZE                = 5000
     MAX_RECONNECT_DELAY                = 15
     REPLY_TIMEOUT                      = 10
     RESOLUTION_FAILURES_BEFORE_WAITING = 3
     RESOLUTION_WAIT                    = 30
     RESOLVE_TIMEOUT                    = 1
+    DEFAULT_FREQUENCY                  = 0
+    VALID_FREQUENCIES                  = [0, 1, 2, 3, 4, 5, 6, 10, 12, 15, 20, 30, 60]
 
 
-    attr_accessor :host, :port, :synchronous, :queue, :dns_resolutions, :last_connect_at
+    attr_accessor :host, :port, :synchronous, :frequency, :sender_queue, :aggregator_queue, :dns_resolutions, :last_connect_at
     attr_reader :connection, :enabled, :secure
 
     def self.logger=(l)
@@ -52,6 +57,7 @@ module Instrumental
       # port:        8001
       # enabled:     true
       # synchronous: false
+      # frequency:   10
       # secure:      true
       # verify:      true
       @api_key         = api_key
@@ -73,13 +79,23 @@ module Instrumental
       @port            = (@port || default_port).to_i
       @enabled         = options.has_key?(:enabled) ? !!options[:enabled] : true
       @synchronous     = !!options[:synchronous]
+
+      if options.has_key?(:frequency)
+        self.frequency = options[:frequency]
+      else
+        self.frequency = DEFAULT_FREQUENCY
+      end
+
+      @metrician       = options[:metrician].nil? ? true : !!options[:metrician]
       @pid             = Process.pid
       @allow_reconnect = true
       @dns_resolutions = 0
       @last_connect_at = 0
-      @metrician       = options[:metrician].nil? ? true : !!options[:metrician]
+
       @start_worker_mutex = Mutex.new
-      @queue = Queue.new
+      @aggregator_queue = Queue.new
+      @sender_queue = Queue.new
+
 
       setup_cleanup_at_exit if @enabled
 
@@ -93,7 +109,9 @@ module Instrumental
     #  agent.gauge('load', 1.23)
     def gauge(metric, value, time = Time.now, count = 1)
       if valid?(metric, value, time, count) &&
-          send_command("gauge", metric, value, time.to_i, count.to_i)
+         send_command(Instrumental::Command.new("gauge".freeze, metric, value, time, count))
+        # tempted to "gauge" this to a symbol? Don't. Frozen strings are very fast,
+        # and later we're going to to_s every one of these anyway.
         value
       else
         nil
@@ -141,7 +159,7 @@ module Instrumental
     #  agent.increment('users')
     def increment(metric, value = 1, time = Time.now, count = 1)
       if valid?(metric, value, time, count) &&
-          send_command("increment", metric, value, time.to_i, count.to_i)
+         send_command(Instrumental::Command.new("increment".freeze, metric, value, time, count))
         value
       else
         nil
@@ -156,7 +174,7 @@ module Instrumental
     #  agent.notice('A notice')
     def notice(note, time = Time.now, duration = 0)
       if valid_note?(note)
-        send_command("notice", time.to_i, duration.to_i, note)
+        send_command(Instrumental::Notice.new(note, time, duration))
         note
       else
         nil
@@ -195,6 +213,22 @@ module Instrumental
       @logger || self.class.logger
     end
 
+    def frequency=(frequency)
+      freq = frequency.to_i
+      if !VALID_FREQUENCIES.include?(freq)
+        logger.warn "Frequency must be a value that divides evenly into 60: 1, 2, 3, 4, 5, 6, 10, 12, 15, 20, 30, or 60."
+        # this will make all negative numbers and nils into 0s
+        freq = VALID_FREQUENCIES.select{ |f| f < freq }.max.to_i
+      end
+
+      @frequency = if(@synchronous)
+                     logger.warn "Synchronous and Frequency should not be enabled at the same time! Defaulting to synchronous mode."
+                     0
+                   else
+                     freq
+                   end
+    end
+
     # Stopping the agent will immediately stop all communication
     # to Instrumental.  If you call this and submit another metric,
     # the agent will start again.
@@ -206,12 +240,19 @@ module Instrumental
     #
     def stop
       disconnect
-      if @thread
-        @thread.kill
-        @thread = nil
+      if @sender_thread
+        @sender_thread.kill
+        @sender_thread = nil
+      end
+      if @aggregator_thread
+        @aggregator_thread.kill
+        @aggregator_thread = nil
       end
-      if @queue
-        @queue.clear
+      if @sender_queue
+        @sender_queue.clear
+      end
+      if @aggregator_queue
+        @aggregator_queue.clear
       end
     end
 
@@ -221,15 +262,22 @@ module Instrumental
     # where at_exit is bypassed like Resque workers.
     def cleanup
       if running?
-        logger.info "Cleaning up agent, queue size: #{@queue.size}, thread running: #{@thread.alive?}"
+        logger.info "Cleaning up agent, aggregator_size: #{@aggregator_queue.size}, thread_running: #{@aggregator_thread.alive?}"
+        logger.info "Cleaning up agent, queue size: #{@sender_queue.size}, thread running: #{@sender_thread.alive?}"
         @allow_reconnect = false
-        if @queue.size > 0
-          queue_message('exit')
+        if @sender_queue.size > 0 || @aggregator_queue.size > 0
+          @sender_queue << ['exit']
+          @aggregator_queue << ['exit']
           begin
-            with_timeout(EXIT_FLUSH_TIMEOUT) { @thread.join }
+            with_timeout(EXIT_FLUSH_TIMEOUT) { @aggregator_thread.join }
+            with_timeout(EXIT_FLUSH_TIMEOUT) { @sender_thread.join }
           rescue Timeout::Error
-            if @queue.size > 0
-              logger.error "Timed out working agent thread on exit, dropping #{@queue.size} metrics"
+            total_size = @sender_queue&.size.to_i +
+                         @aggregator_queue&.size.to_i +
+                         @event_aggregator&.size.to_i
+
+            if total_size > 0
+              logger.error "Timed out working agent thread on exit, dropping #{total_size} metrics"
             else
               logger.error "Timed out Instrumental Agent, exiting"
             end
@@ -270,6 +318,7 @@ module Instrumental
     end
 
     def report_exception(e)
+      # puts "--- Exception of type #{e.class} occurred:\n#{e.message}\n#{e.backtrace.join("\n")}"
       logger.error "Exception of type #{e.class} occurred:\n#{e.message}\n#{e.backtrace.join("\n")}"
     end
 
@@ -290,44 +339,41 @@ module Instrumental
       nil
     end
 
-    def send_command(cmd, *args)
-      cmd = "%s %s\n" % [cmd, args.collect { |a| a.to_s }.join(" ")]
-      if enabled?
-
-        start_connection_worker
-        if @queue && @queue.size < MAX_BUFFER
-          @queue_full_warning = false
-          logger.debug "Queueing: #{cmd.chomp}"
-          queue_message(cmd, { :synchronous => @synchronous })
-        else
-          if !@queue_full_warning
-            @queue_full_warning = true
-            logger.warn "Queue full(#{@queue.size}), dropping commands..."
-          end
-          logger.debug "Dropping command, queue full(#{@queue.size}): #{cmd.chomp}"
-          nil
-        end
+    def send_command(command)
+      return logger.debug(command.to_s) unless enabled?
+      start_workers
+      critical_queue = frequency.to_i == 0 ? @sender_queue : @aggregator_queue
+      if critical_queue && critical_queue.size < MAX_BUFFER
+        @queue_full_warning = false
+        logger.debug "Queueing: #{command.to_s}"
+        queue_message(command, { :synchronous => @synchronous })
       else
-        logger.debug cmd.strip
+        if !@queue_full_warning
+          @queue_full_warning = true
+          logger.warn "Queue full(#{critical_queue.size}), dropping commands..."
+        end
+        logger.debug "Dropping command, queue full(#{critical_queue.size}): #{command.to_s}"
+        nil
       end
     end
 
     def queue_message(message, options = {})
-      if @enabled
-        options ||= {}
-        if options[:allow_reconnect].nil?
-          options[:allow_reconnect] = @allow_reconnect
-        end
-        synchronous = options.delete(:synchronous)
-        if synchronous
-          options[:sync_resource] ||= ConditionVariable.new
-          @sync_mutex.synchronize {
-            @queue << [message, options]
-            options[:sync_resource].wait(@sync_mutex)
-          }
-        else
-          @queue << [message, options]
-        end
+      return message unless enabled?
+
+      # imagine it's a reverse merge, but with fewer allocations
+      options[:allow_reconnect] = @allow_reconnect unless options.has_key?(:allow_reconnect)
+
+      if options.delete(:synchronous)
+        options[:sync_resource] ||= ConditionVariable.new
+        @sync_mutex.synchronize {
+          queue = message == "flush" ? @aggregator_queue : @sender_queue
+          queue << [message, options]
+          options[:sync_resource].wait(@sync_mutex)
+        }
+      elsif frequency.to_i == 0
+        @sender_queue << [message, options]
+      else
+        @aggregator_queue << [message, options]
       end
       message
     end
@@ -355,9 +401,9 @@ module Instrumental
       end
     end
 
-    def start_connection_worker
+    def start_workers
       # NOTE: We need a mutex around both `running?` and thread creation,
-      # otherwise we could create two threads.
+      # otherwise we could create too many threads.
       # Return early and queue the message if another thread is
       # starting the worker.
       return if !@start_worker_mutex.try_lock
@@ -371,9 +417,19 @@ module Instrumental
           @sync_mutex = Mutex.new
           @failures = 0
           @sockaddr_in = Socket.pack_sockaddr_in(@port, address)
-          logger.info "Starting thread"
-          @thread = Thread.new do
-            run_worker_loop
+
+          logger.info "Starting aggregator thread"
+          if !@aggregator_thread&.alive?
+            @aggregator_thread = Thread.new do
+              run_aggregator_loop
+            end
+          end
+
+          if !@sender_thread&.alive?
+            logger.info "Starting sender thread"
+            @sender_thread = Thread.new do
+              run_sender_loop
+            end
           end
         end
       ensure
@@ -409,12 +465,73 @@ module Instrumental
       sock
     end
 
-    def run_worker_loop
+    def run_aggregator_loop
+      # if the sender queue is some level of full, should we keep aggregating until it empties out?
+      # what does this mean for aggregation slices - aggregating to nearest frequency will
+      # make the object needlessly larger, when minute resolution is what we have on the server
+      begin
+        loop do
+          now = Time.now.to_i
+          time_to_wait = if frequency == 0
+                           0
+                         else
+                           next_frequency = (now - (now % frequency)) + frequency
+                           time_to_wait = [(next_frequency - Time.now.to_f), 0].max
+                         end
+
+          command_and_args, command_options = if @event_aggregator&.size.to_i > MAX_AGGREGATOR_SIZE
+                                                logger.info "Aggregator full, flushing early with #{MAX_AGGREGATOR_SIZE} metrics."
+                                                command_and_args, command_options = ['forward', {}]
+                                              else
+                                                begin
+                                                  with_timeout(time_to_wait) do
+                                                    @aggregator_queue.pop
+                                                  end
+                                                rescue Timeout::Error
+                                                  ['forward', {}]
+                                                end
+                                              end
+          if command_and_args
+            sync_resource = command_options && command_options[:sync_resource]
+            case command_and_args
+            when 'exit'
+              logger.info "Exiting, #{@aggregator_queue.size} commands remain"
+              return true
+            when 'flush'
+              if !@event_aggregator.nil?
+                @sender_queue << @event_aggregator
+                @event_aggregator = nil
+              end
+              @sender_queue << ['flush', command_options]
+            when 'forward'
+              if !@event_aggregator.nil?
+                next if @sender_queue.size > 0 && @sender_queue.num_waiting < 1
+                @sender_queue << @event_aggregator
+                @event_aggregator = nil
+              end
+            when Notice
+              @sender_queue << [command_and_args, command_options]
+            else
+              @event_aggregator = EventAggregator.new(frequency: @frequency) if @event_aggregator.nil?
+
+              logger.debug "Sending: #{command_and_args} to aggregator"
+              @event_aggregator.put(command_and_args)
+            end
+            command_and_args = nil
+            command_options = nil
+          end
+        end
+      rescue Exception => err
+        report_exception(err)
+      end
+    end
+
+    def run_sender_loop
       @failures = 0
       begin
-      logger.info "connecting to collector"
-      command_and_args = nil
-      command_options = nil
+        logger.info "connecting to collector"
+        command_and_args = nil
+        command_options = nil
         with_timeout(CONNECT_TIMEOUT) do
           @socket = open_socket(@sockaddr_in, @secure, @verify_cert)
         end
@@ -431,18 +548,23 @@ module Instrumental
         send_with_reply_timeout "authenticate #{@api_key}"
 
         loop do
-          command_and_args, command_options = @queue.pop
+          command_and_args, command_options = @sender_queue.pop
           if command_and_args
             sync_resource = command_options && command_options[:sync_resource]
             test_connection
             case command_and_args
             when 'exit'
-              logger.info "Exiting, #{@queue.size} commands remain"
+              logger.info "Exiting, #{@sender_queue.size} commands remain"
               return true
             when 'flush'
               release_resource = true
+            when EventAggregator
+              command_and_args.values.values.each do |command|
+                logger.debug "Sending: #{command}"
+                @socket.puts command
+              end
             else
-              logger.debug "Sending: #{command_and_args.chomp}"
+              logger.debug "Sending: #{command_and_args}"
               @socket.puts command_and_args
             end
             command_and_args = nil
@@ -464,7 +586,7 @@ module Instrumental
           # or we cannot reach the server
           # or the connection state of this socket is in a race
           # or SSL is not functioning properly for some reason
-          logger.error "unable to connect to Instrumental, hanging up with #{@queue.size} messages remaining"
+          logger.error "unable to connect to Instrumental, hanging up with #{@sender_queue.size} messages remaining"
           logger.debug "Exception: #{err.inspect}\n#{err.backtrace.join("\n")}"
           allow_reconnect = false
         else
@@ -478,7 +600,7 @@ module Instrumental
         end
         if command_and_args
           logger.debug "requeueing: #{command_and_args}"
-          @queue << command_and_args
+          @sender_queue << command_and_args
         end
         disconnect
         @failures += 1
@@ -498,7 +620,11 @@ module Instrumental
     end
 
     def running?
-      !@thread.nil? && @pid == Process.pid && @thread.alive?
+      !@sender_thread.nil? &&
+        !@aggregator_thread.nil? &&
+        @pid == Process.pid &&
+        @sender_thread.alive? &&
+        @aggregator_thread.alive?
     end
 
     def flush_socket(socket)

data/lib/instrumental/command_structs.rb

@@ -0,0 +1,32 @@
+module Instrumental
+  METRIC_TYPES = ["increment".freeze, "gauge".freeze].freeze
+
+  Command = Struct.new(:command, :metric, :value, :time, :count) do
+    def initialize(command, metric, value, time, count)
+      super(command, metric, value, time.to_i, count.to_i)
+    end
+
+    def to_s
+      [command, metric, value, time, count].map(&:to_s).join(" ")
+    end
+
+    def metadata
+      "#{metric}:#{time}".freeze
+    end
+
+    def +(other_command)
+      return self if other_command.nil?
+      Command.new(command, metric, value + other_command.value, time, count + other_command.count)
+    end
+  end
+
+  Notice = Struct.new(:note, :time, :duration) do
+    def initialize(note, time, duration)
+      super(note, time.to_i, duration.to_i)
+    end
+
+    def to_s
+      ["notice".freeze, time, duration, note].map(&:to_s).join(" ")
+    end
+  end
+end

data/lib/instrumental/event_aggregator.rb

@@ -0,0 +1,26 @@
+class EventAggregator
+  attr_accessor :counts, :values, :received_at, :frequency
+ 
+  def initialize(frequency:)
+    @values = Hash.new
+    @frequency = frequency
+  end
+
+  def put(command)
+    command_at = command.time
+    unless(command_at % frequency == 0)
+      command.time = (command_at - (command_at % frequency))
+    end
+    metadata = command.metadata
+    @values[metadata] = (command + @values[metadata])
+  end
+
+  def size
+    @values.size
+  end
+
+  def coerce_time(time)
+    itime = time.to_i
+    (itime - (itime % frequency)).to_i
+  end
+end

data/lib/instrumental/version.rb

@@ -1,3 +1,3 @@
 module Instrumental
-  VERSION = "3.0.0.alpha"
+  VERSION = "3.0.0.beta"
 end

data/spec/agent_spec.rb

@@ -39,7 +39,8 @@ shared_examples "Instrumental Agent" do
     let(:token)        { 'test_token' }
     let(:address)      { server.host_and_port }
     let(:metrician)    { false }
-    let(:agent)        { Instrumental::Agent.new(token, :collector => address, :synchronous => synchronous, :enabled => enabled, :secure => secure?, :verify_cert => verify_cert?, :metrician => metrician) }
+    let(:frequency)    { 0 }
+    let(:agent)        { Instrumental::Agent.new(token, :collector => address, :synchronous => synchronous, :enabled => enabled, :secure => secure?, :verify_cert => verify_cert?, :metrician => metrician, :frequency => frequency) }
 
     # Server options
     let(:listen)       { true }
@@ -47,6 +48,12 @@ shared_examples "Instrumental Agent" do
     let(:authenticate) { true }
     let(:server)       { TestServer.new(:listen => listen, :authenticate => authenticate, :response => response, :secure => secure?) }
 
+    # Time Travel Options
+    let(:start_of_minute) do
+      now = Time.now.to_i
+      Time.at(now - (now % 60))
+    end
+
     before do
       Instrumental::Agent.logger.level = Logger::UNKNOWN
       @server = server
@@ -233,11 +240,11 @@ shared_examples "Instrumental Agent" do
             end
 
             wait
-            expect(agent.queue.size).to eq(3)
-            expect(agent.queue.pop.first).to start_with("increment overflow_test 1 300 1")
-            expect(agent.queue.pop.first).to start_with("increment overflow_test 2 300 1")
-            expect(agent.queue.pop.first).to start_with("increment overflow_test 3 300 1")
-            expect(agent.queue.size).to eq(0)
+            expect(agent.sender_queue.size).to eq(3)
+            expect(agent.sender_queue.pop.first.to_s).to start_with("increment overflow_test 1 300 1")
+            expect(agent.sender_queue.pop.first.to_s).to start_with("increment overflow_test 2 300 1")
+            expect(agent.sender_queue.pop.first.to_s).to start_with("increment overflow_test 3 300 1")
+            expect(agent.sender_queue.size).to eq(0)
           end
         end
       end
@@ -248,7 +255,7 @@ shared_examples "Instrumental Agent" do
           5.times do |i|
             agent.increment('overflow_test', i + 1, 300)
           end
-          expect(agent.instance_variable_get(:@queue).size).to eq(0)
+          expect(agent.instance_variable_get(:@sender_queue).size).to eq(0)
           wait # let the server receive the commands
           expect(server.commands).to include("increment overflow_test 1 300 1")
           expect(server.commands).to include("increment overflow_test 2 300 1")
@@ -264,8 +271,10 @@ shared_examples "Instrumental Agent" do
           fork do
             agent.increment('fork_reconnect_test', 1, 3) # triggers reconnect
           end
+
           wait(1)
           agent.increment('fork_reconnect_test', 1, 4) # triggers reconnect
+
           wait(1)
           expect(server.connect_count).to eq(2)
 
@@ -281,17 +290,17 @@ shared_examples "Instrumental Agent" do
           sleep 1
         }
 
-        run_worker_loop_calls = 0
-        allow(agent).to receive(:run_worker_loop) {
-          run_worker_loop_calls += 1
+        run_sender_loop_calls = 0
+        allow(agent).to receive(:run_sender_loop) {
+          run_sender_loop_calls += 1
           sleep 3 # keep the worker thread alive
         }
 
         t = Thread.new { agent.increment("race") }
         agent.increment("race")
         wait(2)
-        expect(run_worker_loop_calls).to eq(1)
-        expect(agent.queue.size).to eq(2)
+        expect(run_sender_loop_calls).to eq(1)
+        expect(agent.sender_queue.size).to eq(2)
       end
 
       it "should never let an exception reach the user" do
@@ -314,14 +323,6 @@ shared_examples "Instrumental Agent" do
         expect(agent.increment("test")).to eq(nil)
       end
 
-      it "should track invalid metrics" do
-        expect(agent.logger).to receive(:warn).with(/%%/)
-        agent.increment(' %% .!#@$%^&*', 1, 1)
-        wait do
-          expect(server.commands.join("\n")).to include("increment agent.invalid_metric")
-        end
-      end
-
       it "should allow reasonable metric names" do
         agent.increment('a')
         agent.increment('a.b')
@@ -399,9 +400,9 @@ shared_examples "Instrumental Agent" do
 
       it "should allow flushing pending values to the server" do
         1.upto(100) { agent.gauge('a', rand(50)) }
-        expect(agent.instance_variable_get(:@queue).size).to be > 0
+        expect(agent.instance_variable_get(:@sender_queue).size).to be > 0
         agent.flush
-        expect(agent.instance_variable_get(:@queue).size).to eq(0)
+        expect(agent.instance_variable_get(:@sender_queue).size).to eq(0)
         wait do
           expect(server.commands.grep(/^gauge a /).size).to eq(100)
         end
@@ -439,7 +440,7 @@ shared_examples "Instrumental Agent" do
           agent.increment('reconnect_test', 1, 1234)
           wait
           # The agent should not have sent the metric yet, the server is not responding
-          expect(agent.queue.pop(true)).to include("increment reconnect_test 1 1234 1\n")
+          expect(agent.sender_queue.pop(true).first.to_s).to eq("increment reconnect_test 1 1234 1")
         end
 
         it "should warn once when buffer is full" do
@@ -474,7 +475,7 @@ shared_examples "Instrumental Agent" do
           agent.increment('reconnect_test', 1, 1234)
           wait
           # Since server hasn't responded to hello or authenticate, worker thread will not send data
-          expect(agent.queue.pop(true)).to include("increment reconnect_test 1 1234 1\n")
+          expect(agent.sender_queue.pop(true).first.to_s).to eq("increment reconnect_test 1 1234 1")
         end
       end
 
@@ -495,7 +496,7 @@ shared_examples "Instrumental Agent" do
           wait do
             expect(agent.send(:running?)).to eq(false)
           end
-          expect(agent.queue.size).to eq(1)
+          expect(agent.sender_queue.size).to eq(1)
         end
 
         it "should restart the worker thread after hanging it up during an unreachable host event" do
@@ -514,7 +515,7 @@ shared_examples "Instrumental Agent" do
           wait do
             expect(agent.send(:running?)).to eq(false)
           end
-          expect(agent.queue.size).to eq(1)
+          expect(agent.sender_queue.size).to eq(1)
           # Start the server back up again
           server.listen
           # Sending another metric should kickstart the background worker thread
@@ -522,7 +523,7 @@ shared_examples "Instrumental Agent" do
           # The agent should now be running the background thread, and the queue should be empty
           wait do
             expect(agent.send(:running?)).to eq(true)
-            expect(agent.queue.size).to eq(0)
+            expect(agent.sender_queue.size).to eq(0)
           end
         end
 
@@ -547,7 +548,7 @@ shared_examples "Instrumental Agent" do
             expect(agent.send(:running?)).to eq(false)
           end
           # The command is not in the queue
-          expect(agent.queue.size).to eq(0)
+          expect(agent.sender_queue.size).to eq(0)
           # allow the agent to behave normally
           test_connection_fail = false
           # Sending another metric should kickstart the background worker thread
@@ -555,7 +556,7 @@ shared_examples "Instrumental Agent" do
           # The agent should now be running the background thread, and the queue should be empty
           wait do
             expect(agent.send(:running?)).to eq(true)
-            expect(agent.queue.size).to eq(0)
+            expect(agent.sender_queue.size).to eq(0)
             expect(server.commands.grep(/connection_failure/).size).to eq(2)
           end
         end
@@ -580,7 +581,7 @@ shared_examples "Instrumental Agent" do
           agent.gauge('connection_failure_3', 1, 1234)
           wait do
             expect(agent.instance_variable_get(:@failures)).to be > 0
-            expect(agent.queue.size).to be > 0
+            expect(agent.sender_queue.size).to be > 0
           end
 
           # let the loop proceed
@@ -588,12 +589,11 @@ shared_examples "Instrumental Agent" do
 
           wait do
             expect(agent.send(:running?)).to eq(true)
-            expect(agent.queue.size).to eq(0)
+            expect(agent.sender_queue.size).to eq(0)
           end
         end
       end
 
-
       context 'not authenticating' do
         # Server will fail all authentication attempts
         let(:authenticate) { false }
@@ -602,7 +602,7 @@ shared_examples "Instrumental Agent" do
           agent.increment('reconnect_test', 1, 1234)
           wait
           # Metrics should not have been sent since all authentication failed
-          expect(agent.queue.pop(true)).to include("increment reconnect_test 1 1234 1\n")
+          expect(agent.sender_queue.pop(true).first.to_s).to eq("increment reconnect_test 1 1234 1")
         end
       end
 
@@ -639,7 +639,7 @@ shared_examples "Instrumental Agent" do
         it "should not wait to exit a process if there are no commands queued" do
           allow(agent).to receive(:open_socket) { |*args, &block| sleep(5) && block.call }
           with_constants('Instrumental::Agent::EXIT_FLUSH_TIMEOUT' => 3) do
-            if (pid = fork { agent.increment('foo', 1); agent.queue.clear })
+            if (pid = fork { agent.increment('foo', 1); agent.sender_queue.clear })
               tm = Time.now.to_f
               Process.wait(pid)
               diff = Time.now.to_f - tm
@@ -725,7 +725,7 @@ shared_examples "Instrumental Agent" do
           expect(agent.send(:running?)).to eq(true)
 
           # Setup a failure for the next command so we'll break out of the inner
-          # loop in run_worker_loop causing another call to open_socket
+          # loop in run_sender_loop causing another call to open_socket
           test_connection_fail = true
           tc = agent.method(:test_connection)
           allow(agent).to receive(:test_connection) { |*args, &block| test_connection_fail ? raise("fail") : tc.call(*args) }
@@ -802,6 +802,303 @@ shared_examples "Instrumental Agent" do
         end
       end
     end
+
+    describe Instrumental::Agent, "aggregation" do
+      context "aggregation enabled" do
+        let(:frequency) { 2 }
+
+        it "can be enabled at Agent.new time" do
+          expect(agent.frequency).to eq(2)
+        end
+
+        it "can be modified by setting the agent frequency" do
+          agent.frequency = 15
+          expect(agent.frequency).to eq(15)
+        end
+
+        it "is disabled by default" do
+          agent = Instrumental::Agent.new('test_token')
+          expect(agent.frequency.to_f).to eq(0)
+        end
+
+        it "should only allow frequencies that align with minutes" do
+          (-5..100).each do |freq|
+            agent.frequency = freq
+            expect(Instrumental::Agent::VALID_FREQUENCIES).to include(agent.frequency)
+          end
+        end
+
+        it "bypasses aggregator queue entirely for most commands when frequency == 0" do
+          agent.frequency = 0 # this is red - 0 for green
+          expect(EventAggregator).not_to receive(:new)
+          agent.increment('a_metric')
+        end
+
+        it "adds data to the event aggregator and does not immediately send it" do
+          Timecop.travel start_of_minute
+          agent.increment('test')
+          wait do
+            expect(agent.instance_variable_get(:@event_aggregator).size).to eq(1)
+            expect(agent.instance_variable_get(:@event_aggregator).values.values.first.metric).to eq('test')
+          end
+        end
+
+        it "batches data before sending" do
+          Timecop.freeze do
+            agent.increment('a_metric')
+            agent.increment('a_metric')
+            agent.increment('another_metric')
+          end
+          agent.flush(true)
+          wait do
+            expect(server.commands.grep(/_metric/).size).to eq(2)
+            aggregated_metric = server.commands.grep(/a_metric/).first.split(" ")
+            expect(aggregated_metric[2].to_i).to eq(2) # value
+            expect(aggregated_metric[4].to_i).to eq(2) # count
+          end
+        end
+
+        it "aggregates to the specified frequency within the aggregator" do
+          Timecop.travel(start_of_minute)
+          agent.frequency = 15
+          expect(agent.frequency).not_to be(Instrumental::Agent::DEFAULT_FREQUENCY)
+          agent.increment('metric', 1, Time.at(0))
+
+          # will get aligned to the closest frequency (15)
+          agent.increment('metric', 1, Time.at(20))
+          wait do
+            expect(agent.instance_variable_get(:@event_aggregator).values.keys).to eq(["metric:0", "metric:15"])
+          end
+          agent.flush
+          wait do
+            expect(server.commands.grep(/metric 1 0/).size).to eq(1)
+            expect(server.commands.grep(/metric 1 15/).size).to eq(1)
+          end
+        end
+
+        it "flushes data from both queues before sending" do
+          Timecop.freeze do
+            100.times do |i|
+              agent.increment("test_metric_#{i}")
+              agent.increment("other_metric")
+            end
+          end
+
+          expect(agent.instance_variable_get(:@aggregator_queue).size).to be > 0
+          agent.flush
+          expect(agent.instance_variable_get(:@sender_queue).size).to eq(0)
+          expect(agent.instance_variable_get(:@aggregator_queue).size).to eq(0)
+
+          wait do
+            expect(server.commands.grep(/test_metric/).size).to eq(100)
+            expect(server.commands.grep(/other_metric/).size).to eq(1)
+          end
+        end
+
+        it "does not batch notices" do
+          agent.frequency = 60
+          agent.notice "things are happening", 0, 100
+          agent.notice "things are happening", 0, 100
+          agent.notice "things are happening", 0, 100
+          wait do
+            expect(server.commands.grep(/things are happening/).size).to eq(3)
+          end
+        end
+
+        it "can be disabled by setting frequency to nil" do
+          agent.frequency = nil
+          expect(EventAggregator).not_to receive(:new)
+          agent.increment('metric')
+          wait do
+            expect(server.commands.grep(/metric/).size).to eq(1)
+          end
+        end
+
+        it "can be disabled by setting frequency to 0" do
+          agent.frequency = 0
+          expect(EventAggregator).not_to receive(:new)
+          agent.increment('metric')
+          wait do
+            expect(server.commands.grep(/metric/).size).to eq(1)
+          end
+        end
+
+        it "automatically uses the highest-without-going-over frequency for a bad frequency" do
+          agent.frequency = 17
+          expect(agent.frequency).to eq(15)
+          agent.frequency = 69420
+          expect(agent.frequency).to eq(60)
+          agent.frequency = 0
+          expect(agent.frequency).to eq(0)
+          agent.frequency = -1
+          expect(agent.frequency).to eq(0)
+        end
+
+        it "can take strings as frequency" do
+          agent = Instrumental::Agent.new('test_token', :frequency => "15")
+          expect(agent.frequency).to eq(15)
+        end
+
+        it "should not be enabled at the same time as synchronous" do
+          expect(Instrumental::Agent.logger).to receive(:warn).with(/Synchronous and Frequency should not be enabled at the same time! Defaulting to synchronous mode./)
+          agent = Instrumental::Agent.new('test_token', :synchronous => true, :frequency => 6)
+          expect(agent.synchronous).to eq(true)
+          expect(agent.frequency).to eq(0)
+        end
+
+        it "should use synchronous mode if it is enabled, even if turned on after frequency set at start" do
+          agent.increment('metric')
+          agent.increment('metric')
+          agent.synchronous = true
+          agent.increment('metric')
+          wait do
+            expect(server.commands.grep(/metric 1/).size).to eq(1)
+          end
+          agent.flush
+          wait do
+            expect(server.commands.grep(/metric 1/).size).to eq(1)
+            expect(server.commands.grep(/metric 2/).size).to eq(1)
+          end
+        end
+
+        it "sends aggregated metrics after specified frequency, even if no flush is sent" do
+          agent.frequency = 1
+          Timecop.travel(start_of_minute)
+          agent.increment('metric')
+          agent.increment('metric')
+          agent.gauge('other', 1)
+          agent.gauge('other', 1)
+          agent.gauge('other', 1)
+          sleep (0.5)
+          wait { expect(server.commands.grep(/metric/).size).to eq(0) }
+          sleep (0.51) # total sleep > 1 frequency
+
+          expect(server.commands.grep(/metric 2/).size).to eq(1)
+          expect(server.commands.grep(/other 3/).size).to eq(1)
+        end
+
+        # this test really relies on the worker threads not working unexpectedly
+        it "will overflow if the aggregator queue is full" do
+          Timecop.travel(start_of_minute)
+          with_constants('Instrumental::Agent::MAX_BUFFER' => 3) do
+            allow(agent.logger).to receive(:debug)
+            expect(agent.logger).to receive(:debug).with("Dropping command, queue full(3): increment overflow_test 4 300 1")
+            agent.increment('overflow_test', 4, 300, 1)
+            agent.increment('overflow_test', 4, 300, 1)
+            agent.increment('overflow_test', 4, 300, 1)
+            agent.increment('overflow_test', 4, 300, 1)
+
+            expect(agent.instance_variable_get(:@aggregator_queue).size).to eq(3)
+            agent.flush
+            expect(agent.instance_variable_get(:@aggregator_queue).size).to eq(0)
+          end
+        end
+
+        it "if aggregator is at max size, next command will force a forward to the sender thread" do
+          Timecop.travel(start_of_minute)
+          with_constants('Instrumental::Agent::MAX_AGGREGATOR_SIZE' => 3) do
+            agent.increment('overflow_test1')
+            agent.increment('overflow_test2')
+            agent.increment('overflow_test3')
+            agent.increment('overflow_test4')
+            agent.increment('overflow_test5')
+
+            # only 1 because the 5th command triggers a forward of the first 4
+            wait do
+              expect(agent.instance_variable_get(:@event_aggregator).size).to eq(1)
+            end
+            agent.flush
+            wait do
+              expect(server.commands.grep(/overflow_test/).size).to eq(5)
+            end
+          end
+        end
+
+        context do
+          let(:listen) { false }
+          it "will not send aggregators to the sender queue if the sender thread is not ready" do
+            Timecop.travel(start_of_minute)
+            agent.frequency = 1
+
+            with_constants('Instrumental::Agent::MAX_BUFFER' => 3,
+                          'Instrumental::Agent::MAX_AGGREGATOR_SIZE' => 4) do
+
+              # fill the queue
+              agent.increment('overflow_test1')
+              agent.increment('overflow_test2')
+              agent.increment('overflow_test3')
+
+              # wait until they are all in the aggregator
+              wait do
+                expect(agent.instance_variable_get(:@aggregator_queue).size).to eq(0)
+                expect(agent.instance_variable_get(:@event_aggregator).size).to eq(3)
+                expect(agent.instance_variable_get(:@sender_queue).size).to eq(0)
+              end
+
+              # fill the queue again
+              agent.increment('overflow_test1')
+              agent.increment('overflow_test2')
+              agent.increment('overflow_test3')
+
+              # wait until they are all in the aggregator
+              wait do
+                expect(agent.instance_variable_get(:@aggregator_queue).size).to eq(0)
+                expect(agent.instance_variable_get(:@event_aggregator).size).to eq(3)
+                expect(agent.instance_variable_get(:@sender_queue).size).to eq(0)
+              end
+
+              # wait for the aggregator to get forwarded and popped by the sender
+              wait do
+                expect(agent.instance_variable_get(:@aggregator_queue).size).to eq(0)
+                expect(agent.instance_variable_get(:@event_aggregator)).to eq(nil)
+                expect(agent.instance_variable_get(:@sender_queue).size).to eq(1)
+              end
+
+              # fill the queue again
+              agent.increment('overflow_test4')
+              agent.increment('overflow_test5')
+              agent.increment('overflow_test6')
+
+              # wait for them all to be in the aggregator
+              wait do
+                expect(agent.instance_variable_get(:@aggregator_queue).size).to eq(0)
+                expect(agent.instance_variable_get(:@event_aggregator).size).to eq(3)
+                expect(agent.instance_variable_get(:@sender_queue).size).to eq(1)
+              end
+
+              # sleep until the next forward is done
+              sleep(agent.frequency + 0.1)
+
+              # fill the queue again
+              agent.increment('overflow_test7')
+              agent.increment('overflow_test8')
+              agent.increment('overflow_test9')
+
+              # because sending is blocked, the prevous aggregator never sent
+              # when it hits max size, the aggregator queue starts backing up
+              wait do
+                expect(agent.instance_variable_get(:@aggregator_queue).size).to eq(1)
+                expect(agent.instance_variable_get(:@event_aggregator).size).to eq(5)
+                expect(agent.instance_variable_get(:@sender_queue).size).to eq(1)
+              end
+
+              # send 3 more items, to overflow the aggregator queue
+              allow(agent.logger).to receive(:debug)
+              expect(agent.logger).to receive(:debug).with("Dropping command, queue full(3): increment overflow_testc 4 300 1")
+              agent.increment('overflow_testa')
+              agent.increment('overflow_testb')
+              agent.increment('overflow_testc', 4, 300, 1) # will get dropped
+
+              wait do
+                expect(agent.instance_variable_get(:@aggregator_queue).size).to eq(3)
+                expect(agent.instance_variable_get(:@event_aggregator).size).to eq(5)
+                expect(agent.instance_variable_get(:@sender_queue).size).to eq(1)
+              end
+            end
+          end
+        end
+      end
+    end
   end
 end
 

data/spec/command_struct_specs.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+
+describe Instrumental::Command, "basic functions of command structs" do
+  it "should not allow bad arguments to command#+" do
+    command = Instrumental::Command.new("gauge", "abc", 1, Time.at(0), 1)
+
+    # nil is a no-op
+    expect(command + nil).to eq(command)
+    # it will change the top of the other command
+    expect(command + Instrumental::Command.new("increment", "abc", 1, Time.at(0), 1))
+      .to eq(Instrumental::Command.new("gauge", "abc", 2, Time.at(0), 2))
+  end
+
+  it "should add together with like commands" do
+    command = Instrumental::Command.new("gauge", "abc", 1, Time.at(0), 1)
+    other   = Instrumental::Command.new("gauge", "abc", 2, Time.at(0), 4)
+    expect(command + other).to eq(Instrumental::Command.new("gauge", "abc", 3, Time.at(0), 5))
+  end
+end

data/spec/event_aggregator_spec.rb

@@ -0,0 +1,53 @@
+require 'spec_helper'
+
+describe EventAggregator, "time and frequency operations" do
+  it "should massage time values to match the start of a window" do
+    agg = EventAggregator.new(frequency: 10)
+    Timecop.freeze do
+      start_of_minute = Time.now.to_i - (Time.now.to_i % 60)
+      times_to_report = [start_of_minute + 5, start_of_minute + 15]
+      
+      times_to_report.each do |at_time|
+        agg.put(Instrumental::Command.new("gauge", "abc", 5, Time.at(at_time), 1))
+      end
+
+      expect(agg.size).to eq(2)
+
+      expected_values = [Instrumental::Command.new("gauge", "abc", 5, Time.at(start_of_minute), 1),
+                         Instrumental::Command.new("gauge", "abc", 5, Time.at(start_of_minute + 10), 1)]
+      expect(agg.values.values).to eq(expected_values)
+    end
+  end
+end
+
+describe EventAggregator do
+  it "should aggregate put operations to a given frequency" do
+    start_of_minute = Time.now.to_i - (Time.now.to_i % 60)
+    Timecop.freeze(Time.at(start_of_minute)) do
+      agg = EventAggregator.new(frequency: 30)
+      (Time.now.to_i..(Time.now.to_i + 119)).each do |time|
+        agg.put(Instrumental::Command.new("increment", "abc", 1, time, 1))
+      end
+      expect(agg.size).to eq(4)
+      (Time.now.to_i..(Time.now.to_i + 119)).step(30).map do |time|
+        expect(agg.values["abc:#{time}"]).to eq(Instrumental::Command.new("increment", "abc", 30, time, 30))
+      end
+    end
+  end
+
+  it "should aggregate put operations to the same metric and last type wins" do
+    Timecop.freeze do
+      agg = EventAggregator.new(frequency: 6)
+
+      agg.put(Instrumental::Command.new("gauge", "hello", 3.0, Time.now, 1))
+      agg.put(Instrumental::Command.new("increment", "hello", 4.0, Time.now, 1))
+      
+      expect(agg.size).to eq(1)
+      expect(agg.values.values.first).to eq(Instrumental::Command.new("increment",
+                                                                     "hello",
+                                                                     7.0,
+                                                                     agg.coerce_time(Time.now),
+                                                                     2))
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: instrumental_agent
 version: !ruby/object:Gem::Version
-  version: 3.0.0.alpha
+  version: 3.0.0.beta
 platform: ruby
 authors:
 - Expected Behavior
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-08-22 00:00:00.000000000 Z
+date: 2020-10-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: metrician
@@ -118,12 +118,16 @@ files:
 - lib/instrumental/capistrano.rb
 - lib/instrumental/capistrano/capistrano2.rb
 - lib/instrumental/capistrano/capistrano3.rake
+- lib/instrumental/command_structs.rb
+- lib/instrumental/event_aggregator.rb
 - lib/instrumental/system_timer.rb
 - lib/instrumental/version.rb
 - lib/instrumental_agent.rb
 - script/setup
 - script/test
 - spec/agent_spec.rb
+- spec/command_struct_specs.rb
+- spec/event_aggregator_spec.rb
 - spec/spec_helper.rb
 - spec/test.crt
 - spec/test.csr
@@ -133,7 +137,7 @@ homepage: http://github.com/instrumental/instrumental_agent-ruby
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -141,7 +145,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.4.6
+      version: 2.5.7
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">"
@@ -149,11 +153,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubygems_version: 3.0.3
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Custom metric monitoring for Ruby applications via Instrumental
 test_files:
 - spec/agent_spec.rb
+- spec/command_struct_specs.rb
+- spec/event_aggregator_spec.rb
 - spec/spec_helper.rb
 - spec/test.crt
 - spec/test.csr