instrumental_agent 0.3.0 → 0.4.0

data/lib/instrumental/agent.rb

@@ -20,8 +20,8 @@ module Instrumental
       @logger = l
     end
 
-    def self.logger
-      @logger ||= Logger.new('/dev/null')
+    def self.logger(force = false)
+      @logger ||= Logger.new(File.open('/dev/null', 'a')) # append mode so it's forksafe
     end
 
     def self.all
@@ -52,11 +52,14 @@ module Instrumental
       @port      = (collector[1] || 8000).to_i
       @enabled   = options[:enabled]
       @test_mode = options[:test_mode]
+      @pid = Process.pid
+
 
       if @enabled
         @failures = 0
         @queue = Queue.new
-        start_connection_thread
+        start_connection_worker
+        setup_cleanup_at_exit
       end
     end
 
@@ -95,7 +98,7 @@ module Instrumental
     end
 
     def connected?
-      connection && connection.connected
+      @socket && !@socket.closed?
     end
 
     def logger
@@ -125,6 +128,14 @@ module Instrumental
 
     def send_command(cmd, *args)
       if enabled?
+        if @pid != Process.pid
+          logger.info "Detected fork"
+          @pid = Process.pid
+          @socket = nil
+          @queue = Queue.new
+          start_connection_worker
+        end
+
         cmd = "%s %s\n" % [cmd, args.collect(&:to_s).join(" ")]
         if @queue.size < MAX_BUFFER
           logger.debug "Queueing: #{cmd.chomp}"
@@ -137,7 +148,7 @@ module Instrumental
       end
     end
 
-    def test_server_connection
+    def test_connection
       # FIXME: Test connection state hack
       begin
         @socket.read_nonblock(1) # TODO: put data back?
@@ -146,43 +157,57 @@ module Instrumental
       end
     end
 
-    def start_connection_thread
-      logger.info "Starting thread"
-      @thread = Thread.new do
-        begin
-          @socket = TCPSocket.new(host, port)
-          @failures = 0
-          logger.info "connected to collector"
-          @socket.puts "hello version #{Instrumental::VERSION} test_mode #{@test_mode}"
-          @socket.puts "authenticate #{@api_key}"
+    def start_connection_worker
+      if enabled?
+        disconnect
+        logger.info "Starting thread"
+        @thread = Thread.new do
           loop do
-            command_and_args = @queue.pop
-            begin
-              test_server_connection
-            rescue Exception => err
-              @queue << command_and_args # connection dead, requeue
-              raise err
-            end
-
-            if command_and_args == 'exit'
-              logger.info "exiting, #{@queue.size} commands remain"
-              @socket.flush
-              Thread.exit
-            else
-              logger.debug "Sending: #{command_and_args.chomp}"
-              @socket.puts command_and_args
-            end
+            break if connection_worker
           end
-        rescue Exception => err
-          logger.error err.to_s
-          # FIXME: not always a disconnect
-          @failures += 1
-          delay = [(@failures - 1) ** BACKOFF, MAX_RECONNECT_DELAY].min
-          logger.info "disconnected, reconnect in #{delay}..."
-          sleep delay
-          retry
         end
       end
+    end
+
+    def connection_worker
+      command_and_args = nil
+      logger.info "connecting to collector"
+      @socket = TCPSocket.new(host, port)
+      @failures = 0
+      logger.info "connected to collector at #{host}:#{port}"
+      @socket.puts "hello version #{Instrumental::VERSION} test_mode #{@test_mode}"
+      @socket.puts "authenticate #{@api_key}"
+      loop do
+        command_and_args = @queue.pop
+        test_connection
+
+        case command_and_args
+        when 'exit'
+          logger.info "exiting, #{@queue.size} commands remain"
+          return true
+        else
+          logger.debug "Sending: #{command_and_args.chomp}"
+          @socket.puts command_and_args
+          command_and_args = nil
+        end
+      end
+    rescue Exception => err
+      logger.error err.to_s
+      if command_and_args
+        logger.debug "requeueing: #{command_and_args}"
+        @queue << command_and_args 
+      end
+      disconnect
+      @failures += 1
+      delay = [(@failures - 1) ** BACKOFF, MAX_RECONNECT_DELAY].min
+      logger.info "disconnected, reconnect in #{delay}..."
+      sleep delay
+      retry
+    ensure
+      disconnect
+    end
+
+    def setup_cleanup_at_exit
       at_exit do
         if !@queue.empty? && @thread.alive?
           if @failures > 0
@@ -196,6 +221,16 @@ module Instrumental
         end
       end
     end
+
+    def disconnect
+      if connected?
+        logger.info "Disconnecting..."
+        @socket.flush
+        @socket.close
+      end
+      @socket = nil
+    end
+
   end
 
 end

data/lib/instrumental/version.rb

@@ -1,3 +1,3 @@
 module Instrumental
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/spec/agent_spec.rb

@@ -19,7 +19,7 @@ describe Instrumental::Agent, "disabled" do
     @server.connect_count.should == 0
   end
 
-  it "should not connect to the server" do
+  it "should not connect to the server after receiving a metric" do
     wait
     @agent.gauge('disabled_test', 1)
     wait
@@ -149,6 +149,21 @@ describe Instrumental::Agent, "enabled" do
     @server.commands.last.should == "increment reconnect_test 1 1234"
   end
 
+  it "should automatically reconnect when forked" do
+    wait
+    @agent.increment('fork_reconnect_test', 1, 2)
+    fork do
+      @agent.increment('fork_reconnect_test', 1, 3) # triggers reconnect
+    end
+    wait
+    @agent.increment('fork_reconnect_test', 1, 4) # triggers reconnect
+    wait
+    @server.connect_count.should == 2
+    @server.commands.should include("increment fork_reconnect_test 1 2")
+    @server.commands.should include("increment fork_reconnect_test 1 3")
+    @server.commands.should include("increment fork_reconnect_test 1 4")
+  end
+
   it "should never let an exception reach the user" do
     @agent.stub!(:send_command).and_raise(Exception.new("Test Exception"))
     @agent.increment('throws_exception', 2).should be_nil

data/spec/test_server.rb

@@ -5,6 +5,7 @@ class TestServer
     @connect_count = 0
     @connections = []
     @commands = []
+    @host = 'localhost'
     listen
   end
 
@@ -14,18 +15,22 @@ class TestServer
     Thread.new do
       begin
         # puts "listening"
-        socket = @server.accept
-        @connect_count += 1
-        @connections << socket
-        # puts "connection received"
         loop do
-          command = socket.gets.strip
-          # puts "got: #{command}"
-          commands << command
+          socket = @server.accept
+          Thread.new do
+            @connect_count += 1
+            @connections << socket
+            # puts "connection received"
+            loop do
+              command = socket.gets.strip
+              # puts "got: #{command}"
+              commands << command
+            end
+          end
         end
       rescue Exception => err
         unless @stopping
-          # puts "EXCEPTION:", err unless @stopping
+          puts "EXCEPTION:", err unless @stopping
           retry
         end
       end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: instrumental_agent
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 15
   prerelease: 
   segments: 
   - 0
-  - 3
+  - 4
   - 0
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors: 
 - Elijah Miller
@@ -17,7 +17,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-11-17 00:00:00 -05:00
+date: 2011-12-01 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency