instrumental_agent 0.8.3 → 0.9.0

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+### 0.9.0 [February 20th, 2012]
+* Added manual synchronous flushing command
+* Fixed bug with data dropping on short-lived forks
+
 ### 0.8.3 [February 9th, 2012]
 * Removing symbol to proc use for compatibility with older version of Ruby
 

data/Gemfile

@@ -1,3 +1,8 @@
 source :rubygems
 
 gemspec
+
+if RUBY_VERSION < "1.9"
+  # Built and installed via ext/mkrf_conf.rb
+  gem 'system_timer', '~> 1.2'
+end

data/README.md

@@ -111,5 +111,4 @@ after  "instrumental:util:deploy_end", "instrumental:record_deploy_notice"
 
 ## Troubleshooting & Help
 
-We are here to help, please email us at
-[support@instrumentalapp.com](mailto:support@instrumentalapp.com).
+We are here to help. Email us at [support@instrumentalapp.com](mailto:support@instrumentalapp.com), or visit the [Instrumental Support](https://fastestforward.campfirenow.com/6b934) Campfire room.

data/ext/mkrf_conf.rb

@@ -0,0 +1,20 @@
+require 'rubygems'
+require 'rubygems/command.rb'
+require 'rubygems/dependency_installer.rb' 
+begin
+  Gem::Command.build_args = ARGV
+  rescue NoMethodError
+end 
+inst = Gem::DependencyInstaller.new
+begin
+  if RUBY_VERSION < "1.9"
+    inst.install "system_timer", "~> 1.2"
+  end
+rescue
+  puts "Couldn't install system_timer gem, required on Ruby < 1.9"
+  exit(1)
+end 
+
+f = File.open(File.join(File.dirname(__FILE__), "Rakefile"), "w")   # create dummy rakefile to indicate success
+f.write("task :default\n")
+f.close

data/instrumental_agent.gemspec

@@ -14,11 +14,14 @@ Gem::Specification.new do |s|
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
+  s.extensions    = 'ext/mkrf_conf.rb'
   s.add_development_dependency(%q<rake>, [">= 0"])
   s.add_development_dependency(%q<rspec>, ["~> 2.0"])
-  s.add_development_dependency(%q<guard>, [">= 0"])
-  s.add_development_dependency(%q<guard-rspec>, [">= 0"])
-  s.add_development_dependency(%q<growl_notify>, [">= 0"])
-  s.add_development_dependency(%q<rb-fsevent>, [">= 0"])
   s.add_development_dependency(%q<fuubar>, [">= 0"])
+  if RUBY_VERSION >= "1.8.7"
+    s.add_development_dependency(%q<guard>, [">= 0"])
+    s.add_development_dependency(%q<guard-rspec>, [">= 0"])
+    s.add_development_dependency(%q<growl>, [">= 0"])
+    s.add_development_dependency(%q<rb-fsevent>, [">= 0"])
+  end
 end

data/lib/instrumental/agent.rb

@@ -3,7 +3,12 @@ require 'instrumental/version'
 require 'logger'
 require 'thread'
 require 'socket'
-require 'timeout'
+if RUBY_VERSION < "1.9"
+  require 'system_timer'
+else
+  require 'timeout'
+end
+
 
 # Sets up a connection to the collector.
 #
@@ -14,6 +19,8 @@ module Instrumental
     MAX_RECONNECT_DELAY = 15
     MAX_BUFFER = 5000
     REPLY_TIMEOUT = 10
+    CONNECT_TIMEOUT = 20
+    EXIT_FLUSH_TIMEOUT = 5
 
     attr_accessor :host, :port, :synchronous, :queue
     attr_reader :connection, :enabled
@@ -60,12 +67,14 @@ module Instrumental
       @enabled     = options[:enabled]
       @test_mode   = options[:test_mode]
       @synchronous = options[:synchronous]
+      @allow_reconnect = true
       @pid = Process.pid
 
 
       if @enabled
         @failures = 0
         @queue = Queue.new
+        @sync_mutex = Mutex.new
         start_connection_worker
         setup_cleanup_at_exit
       end
@@ -136,7 +145,7 @@ module Instrumental
 
     # Send a notice to the server (deploys, downtime, etc.)
     #
-    # agent.notice('A notice')
+    #  agent.notice('A notice')
     def notice(note, time = Time.now, duration = 0)
       if valid_note?(note)
         send_command("notice", time.to_i, duration.to_i, note)
@@ -149,6 +158,19 @@ module Instrumental
       nil
     end
 
+    # Synchronously flush all pending metrics out to the server
+    # By default will not try to reconnect to the server if a
+    # connection failure happens during the flush, though you
+    # may optionally override this behavior by passing true.
+    #
+    #  agent.flush
+    def flush(allow_reconnect = false)
+      queue_message('flush', {
+        :synchronous => true,
+        :allow_reconnect => allow_reconnect
+      })
+    end
+
     def enabled?
       @enabled
     end
@@ -167,6 +189,11 @@ module Instrumental
 
     private
 
+    def with_timeout(time, &block)
+      tmr_klass = RUBY_VERSION < "1.9" ? SystemTimer : Timeout
+      tmr_klass.timeout(time) { yield }
+    end
+
     def valid_note?(note)
       note !~ /[\n\r]/
     end
@@ -207,13 +234,10 @@ module Instrumental
           start_connection_worker
         end
 
-        cmd = "%s %s\n" % [cmd, args.collect { |v| v.to_s }.join(" ")]
+        cmd = "%s %s\n" % [cmd, args.collect { |a| a.to_s }.join(" ")]
         if @queue.size < MAX_BUFFER
           logger.debug "Queueing: #{cmd.chomp}"
-          @main_thread = Thread.current if @synchronous
-          @queue << cmd
-          Thread.stop if @synchronous
-          cmd
+          queue_message(cmd, { :synchronous => @synchronous })
         else
           logger.warn "Dropping command, queue full(#{@queue.size}): #{cmd.chomp}"
           nil
@@ -221,12 +245,32 @@ module Instrumental
       end
     end
 
+    def queue_message(message, options = {})
+      if @enabled
+        options ||= {}
+        if options[:allow_reconnect].nil?
+          options[:allow_reconnect] = @allow_reconnect
+        end
+        synchronous = options.delete(:synchronous)
+        if synchronous
+          options[:sync_resource] ||= ConditionVariable.new
+          @queue << [message, options]
+          @sync_mutex.synchronize {
+            options[:sync_resource].wait(@sync_mutex)
+          }
+        else
+          @queue << [message, options]
+        end
+      end
+      message
+    end
+
     def test_connection
       # FIXME: Test connection state hack
       begin
         @socket.read_nonblock(1) # TODO: put data back?
       rescue Errno::EAGAIN
-        # nop
+        # noop
       end
     end
 
@@ -235,16 +279,14 @@ module Instrumental
         disconnect
         logger.info "Starting thread"
         @thread = Thread.new do
-          loop do
-            break if connection_worker
-          end
+          run_worker_loop
         end
       end
     end
 
     def send_with_reply_timeout(message)
       @socket.puts message
-      Timeout.timeout(REPLY_TIMEOUT) do
+      with_timeout(REPLY_TIMEOUT) do
         response = @socket.gets
         if response.to_s.chomp != "ok"
           raise "Bad Response #{response.inspect} to #{message.inspect}"
@@ -252,31 +294,44 @@ module Instrumental
       end
     end
 
-    def connection_worker
+    def run_worker_loop
       command_and_args = nil
+      command_options = nil
       logger.info "connecting to collector"
-      @socket = TCPSocket.new(host, port)
+      @socket = with_timeout(CONNECT_TIMEOUT) { TCPSocket.new(host, port) }
       logger.info "connected to collector at #{host}:#{port}"
       send_with_reply_timeout "hello version #{Instrumental::VERSION} test_mode #{@test_mode}"
       send_with_reply_timeout "authenticate #{@api_key}"
       @failures = 0
       loop do
-        command_and_args = @queue.pop
+        command_and_args, command_options = @queue.pop
+        sync_resource = command_options && command_options[:sync_resource]
         test_connection
-
         case command_and_args
         when 'exit'
           logger.info "exiting, #{@queue.size} commands remain"
           return true
+        when 'flush'
+          release_resource = true
         else
           logger.debug "Sending: #{command_and_args.chomp}"
           @socket.puts command_and_args
-          command_and_args = nil
         end
-        @main_thread.run if @synchronous
+        command_and_args = nil
+        command_options = nil
+        if sync_resource
+          @sync_mutex.synchronize do
+            sync_resource.signal
+          end
+        end
       end
     rescue Exception => err
-      logger.error err.to_s
+      logger.debug err.backtrace.join("\n")
+      if @allow_reconnect == false ||
+        (command_options && command_options[:allow_reconnect] == false)
+        logger.error "Not trying to reconnect"
+        return
+      end
       if command_and_args
         logger.debug "requeueing: #{command_and_args}"
         @queue << command_and_args
@@ -293,16 +348,20 @@ module Instrumental
 
     def setup_cleanup_at_exit
       at_exit do
-        if !@queue.empty? && @thread.alive?
-          if @failures > 0
-            logger.info "exit received but disconnected, dropping #{@queue.size} commands"
-            @thread.kill
+        logger.info "Cleaning up agent, queue empty: #{@queue.empty?}, thread running: #{@thread.alive?}"
+        @allow_reconnect = false
+        logger.info "exit received, currently #{@queue.size} commands to be sent"
+        queue_message('exit')
+        begin
+          with_timeout(EXIT_FLUSH_TIMEOUT) { @thread.join }
+        rescue Timeout::Error
+          if @queue.size > 0
+            logger.error "Timed out working agent thread on exit, dropping #{@queue.size} metrics"
           else
-            logger.info "exit received, #{@queue.size} commands to be sent"
-            @queue << 'exit'
-            @thread.join
+            logger.error "Timed out Instrumental Agent, exiting"
           end
         end
+
       end
     end
 

data/lib/instrumental/version.rb

@@ -1,3 +1,3 @@
 module Instrumental
-  VERSION = '0.8.3'
+  VERSION = '0.9.0'
 end

data/spec/agent_spec.rb

@@ -27,6 +27,19 @@ describe Instrumental::Agent, "disabled" do
     @server.connect_count.should == 0
   end
 
+  it "should no op on flush without reconnect" do
+    1.upto(100) { @agent.gauge('disabled_test', 1) }
+    @agent.flush(false)
+    wait
+    @server.commands.should be_empty
+  end
+
+  it "should no op on flush with reconnect" do
+    1.upto(100) { @agent.gauge('disabled_test', 1) }
+    @agent.flush(true)
+    wait
+    @server.commands.should be_empty
+  end
 end
 
 describe Instrumental::Agent, "enabled in test_mode" do
@@ -81,7 +94,7 @@ describe Instrumental::Agent, "enabled in test_mode" do
         throw :an_exception
         sleep 1
       end
-      }.should raise_error
+    }.should raise_error
     wait
     @server.commands.last.should =~ /gauge time_value_test .* #{now.to_i}/
     time = @server.commands.last.scan(/gauge time_value_test (.*) #{now.to_i}/)[0][0].to_f
@@ -216,6 +229,7 @@ describe Instrumental::Agent, "enabled" do
       5.times do |i|
         @agent.increment('overflow_test', i + 1, 300)
       end
+      @agent.instance_variable_get(:@queue).size.should == 0
       wait # let the server receive the commands
       @server.commands.should include("increment overflow_test 1 300")
       @server.commands.should include("increment overflow_test 2 300")
@@ -309,6 +323,15 @@ describe Instrumental::Agent, "enabled" do
     wait
     @server.commands.join("\n").should_not include("notice Test note")
   end
+
+  it "should allow flushing pending values to the server" do
+    1.upto(100) { @agent.gauge('a', rand(50)) }
+    @agent.instance_variable_get(:@queue).size.should >= 100
+    @agent.flush
+    @agent.instance_variable_get(:@queue).size.should ==  0
+    wait
+    @server.commands.grep(/^gauge a /).size.should == 100
+  end
 end
 
 describe Instrumental::Agent, "connection problems" do
@@ -333,7 +356,7 @@ describe Instrumental::Agent, "connection problems" do
     wait
     @agent.increment('reconnect_test', 1, 1234)
     wait
-    @agent.queue.pop(true).should == "increment reconnect_test 1 1234\n"
+    @agent.queue.pop(true).should include("increment reconnect_test 1 1234\n")
   end
 
   it "should buffer commands when server is not responsive" do
@@ -342,7 +365,7 @@ describe Instrumental::Agent, "connection problems" do
     wait
     @agent.increment('reconnect_test', 1, 1234)
     wait
-    @agent.queue.pop(true).should == "increment reconnect_test 1 1234\n"
+    @agent.queue.pop(true).should include("increment reconnect_test 1 1234\n")
   end
 
   it "should buffer commands when authentication fails" do
@@ -351,7 +374,31 @@ describe Instrumental::Agent, "connection problems" do
     wait
     @agent.increment('reconnect_test', 1, 1234)
     wait
-    @agent.queue.pop(true).should == "increment reconnect_test 1 1234\n"
+    @agent.queue.pop(true).should include("increment reconnect_test 1 1234\n")
+  end
+
+  it "should send commands in a short-lived process" do
+    @server = TestServer.new
+    @agent = Instrumental::Agent.new('test_token', :collector => @server.host_and_port, :synchronous => false)
+    if pid = fork { @agent.increment('foo', 1, 1234) }
+      Process.wait(pid)
+      @server.commands.last.should == "increment foo 1 1234"
+    end
+  end
+
+  it "should not wait longer than EXIT_FLUSH_TIMEOUT seconds to exit a process" do
+    @server = TestServer.new
+    @agent = Instrumental::Agent.new('test_token', :collector => @server.host_and_port, :synchronous => false)
+    TCPSocket.stub!(:new) { |*args| sleep(5) && StringIO.new }
+    with_constants('Instrumental::Agent::EXIT_FLUSH_TIMEOUT' => 3) do
+      if (pid = fork { @agent.increment('foo', 1) })
+        tm = Time.now.to_f
+        Process.wait(pid)
+        diff = Time.now.to_f - tm
+        diff.should >= 3
+        diff.should < 5
+      end
+    end
   end
 end
 

data/spec/spec_helper.rb

@@ -15,8 +15,10 @@ end
 
 
 def parse_constant(constant)
-  source, _, constant_name = constant.to_s.rpartition('::')
-
+  constant = constant.to_s
+  parts = constant.split("::")
+  constant_name = parts.pop
+  source = parts.join("::")
   [source.constantize, constant_name]
 end
 

data/spec/test_server.rb

@@ -55,6 +55,7 @@ class TestServer
   rescue Exception => err
     # FIXME: doesn't seem to be detecting failures of listen
     puts "failed to get port"
+    puts err.message
     @port += 1
     retry
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: instrumental_agent
 version: !ruby/object:Gem::Version
-  version: 0.8.3
+  version: 0.9.0
   prerelease: 
 platform: ruby
 authors:
@@ -12,11 +12,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-09 00:00:00.000000000 Z
+date: 2012-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70106588068980 !ruby/object:Gem::Requirement
+  requirement: &70233623514200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -24,10 +24,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70106588068980
+  version_requirements: *70233623514200
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70106588083600 !ruby/object:Gem::Requirement
+  requirement: &70233623513680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -35,10 +35,10 @@ dependencies:
         version: '2.0'
   type: :development
   prerelease: false
-  version_requirements: *70106588083600
+  version_requirements: *70233623513680
 - !ruby/object:Gem::Dependency
-  name: guard
-  requirement: &70106588082500 !ruby/object:Gem::Requirement
+  name: fuubar
+  requirement: &70233623513200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -46,10 +46,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70106588082500
+  version_requirements: *70233623513200
 - !ruby/object:Gem::Dependency
-  name: guard-rspec
-  requirement: &70106588080820 !ruby/object:Gem::Requirement
+  name: guard
+  requirement: &70233623512700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -57,10 +57,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70106588080820
+  version_requirements: *70233623512700
 - !ruby/object:Gem::Dependency
-  name: growl_notify
-  requirement: &70106588078620 !ruby/object:Gem::Requirement
+  name: guard-rspec
+  requirement: &70233623512220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -68,10 +68,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70106588078620
+  version_requirements: *70233623512220
 - !ruby/object:Gem::Dependency
-  name: rb-fsevent
-  requirement: &70106588077720 !ruby/object:Gem::Requirement
+  name: growl
+  requirement: &70233623511740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -79,10 +79,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70106588077720
+  version_requirements: *70233623511740
 - !ruby/object:Gem::Dependency
-  name: fuubar
-  requirement: &70106588077080 !ruby/object:Gem::Requirement
+  name: rb-fsevent
+  requirement: &70233623511260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -90,12 +90,13 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70106588077080
+  version_requirements: *70233623511260
 description: Track anything.
 email:
 - support@instrumentalapp.com
 executables: []
-extensions: []
+extensions:
+- ext/mkrf_conf.rb
 extra_rdoc_files: []
 files:
 - .gitignore
@@ -105,6 +106,7 @@ files:
 - Guardfile
 - README.md
 - Rakefile
+- ext/mkrf_conf.rb
 - instrumental_agent.gemspec
 - lib/instrumental/agent.rb
 - lib/instrumental/capistrano.rb
@@ -128,7 +130,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4247345232071751034
+      hash: -278624242877907037
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -137,7 +139,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4247345232071751034
+      hash: -278624242877907037
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.15