logstash_writer 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 834ed0e5a393419b960245f1d036722ff97412a4009e3586fc9dc7d860fe79e7
-  data.tar.gz: bb9ef3b35721e7768348f74b83bcea4315ab0b6b3f968bc7874118d7d58be318
+  metadata.gz: 3b3f279724a1874b171c9080fb4aae626e844c13b845bec88a50ae832b660a85
+  data.tar.gz: cb3463bfddf9673808fb874cc1bca6b1b51527f2eb626134bada7cacff91b856
 SHA512:
-  metadata.gz: 2a31b62d8a681acc81a21491cf5b2f26807114bc81237e03f79eb5f053238bd3dab8fcc2394548f3546854534a54f6fce234ef622002ced68ed7a642360460e8
-  data.tar.gz: 7b857ce439143902b975132ee2f020220bebcdb4010f0c1bbc611cd114aa1d127df75aa11747af3b0b7776cf94c195c3e829a0d6ee37136bd762143391c3545b
+  metadata.gz: 41611118bbcc2992196a8c2dc49bec74ce00dc3ec0505b9682132c9e6bcf6e7a8dd48355f35f06cd393f8e2c41460a24b28c53b0c0b0f5ee48c8ec6ec9964503
+  data.tar.gz: 5527ef9d6bf2a9ed3de0c3483bf43ed6ffd5e69a4e7b8687bdb9d33d258651bdae95a9b990617007578fb10003533840b6f397d87fa2e955836704f9eb064955

data/README.md

@@ -146,9 +146,9 @@ The metrics that are exposed are:
 
 * **`logstash_writer_connected_to_server`** -- this flag timeseries (can be
   either `1` or `0`) is simply a way for you to quickly determine whether
-  the writer has a server to talk to, if it wants one.  That is, this time
-  series will only be `0` if there's an event to write but no logstash
-  server can be found to write it to.
+  the writer has a server to talk to, if it wants one, and which server it
+  is connected to.  That is, this time series will only be `0` if there's an
+  event to write but no logstash server can be found to write it to.
 
 * **`logstash_writer_connect_exceptions_total`** -- a count of exceptions
   raised whilst attempting to connect to a logstash server, labelled by the

data/lib/logstash_writer.rb

@@ -183,7 +183,7 @@ class LogstashWriter
           @logger.error("LogstashWriter") { (["Worker thread terminated with exception: #{ex.message} (#{ex.class})"] + ex.backtrace).join("\n  ") }
         end
         @worker_thread = nil
-        @socket_mutex.synchronize { (@current_socket.close; @current_socket = nil) if @current_socket }
+        @socket_mutex.synchronize { (@current_target.close; @current_target = nil) if @current_target }
       end
     end
 
@@ -201,11 +201,11 @@ class LogstashWriter
   #
   def force_disconnect!
     @socket_mutex.synchronize do
-      return if @current_socket.nil?
+      return if @current_target.nil?
 
-      @logger.info("LogstashWriter") { "Forced disconnect from #{describe_peer(@current_socket) }" }
-      @current_socket.close if @current_socket
-      @current_socket = nil
+      @logger.info("LogstashWriter") { "Forced disconnect from #{@current_target.describe_peer}" }
+      @current_target.close
+      @current_target = nil
     end
 
     nil
@@ -237,20 +237,14 @@ class LogstashWriter
             event = @queue.shift
           end
 
-          current_socket do |s|
-            s.puts event[:content].to_json
-            stat_sent(describe_peer(s), event[:arrival_timestamp])
+          current_target do |t|
+            t.socket.puts event[:content].to_json
+            stat_sent(t.to_s, event[:arrival_timestamp])
             @metrics[:write_loop_ok].set({}, 1)
             error_wait = INITIAL_RETRY_WAIT
           end
         rescue StandardError => ex
           @logger.error("LogstashWriter") { (["Exception in write_loop: #{ex.message} (#{ex.class})"] + ex.backtrace).join("\n  ") }
-          # If there was some sort of error, there's a non-trivial chance the
-          # socket has gone *boom*, so let's invalidate it and go around again
-          if @current_socket
-            @current_socket.close
-            @current_socket = nil
-          end
           @queue_mutex.synchronize { @queue.unshift(event) if event }
           @metrics[:write_loop_exception].increment(class: ex.class.to_s)
           @metrics[:write_loop_ok].set({}, 0)
@@ -265,14 +259,14 @@ class LogstashWriter
     end
   end
 
-  # Yield a TCPSocket connected to the server we currently believe to be
+  # Yield a Target connected to the server we currently believe to be
   # accepting log entries, so that something can send log entries to it.
   #
   # The yielding allows us to centralise all error detection and handling
   # within this one method, and retry sending just by calling `yield` again
   # when we've connected to another server.
   #
-  def current_socket
+  def current_target
     # This could all be handled more cleanly with recursion, but I don't
     # want to fill the stack if we have to retry a lot of times.  Also
     # can't just use `retry` because not all of the "go around again"
@@ -281,19 +275,25 @@ class LogstashWriter
 
     until done
       @socket_mutex.synchronize do
-        if @current_socket
+        if @current_target
           begin
-            yield @current_socket
-            @metrics[:connected].set({}, 1)
+            # Check that our socket is still good to go; if we don't do this,
+            # the other end can disconnect, and because we're never normally
+            # reading from the socket, we never get the EOFError that normally
+            # results, and so the socket remains in CLOSE_WAIT state *forever*.
+            @current_target.socket.read_nonblock(1)
+
+            yield @current_target
+            @metrics[:connected].set({ server: @current_target.describe_peer }, 1)
             done = true
-          rescue SystemCallError => ex
+          rescue SystemCallError, IOError => ex
             # Something went wrong during the send; disconnect from this
             # server and recycle
-            @metrics[:write_exception].increment(server: describe_peer(@current_socket), class: ex.class.to_s)
-            @logger.info("LogstashWriter") { "Error while writing to current server: #{ex.message} (#{ex.class})" }
-            @current_socket.close
-            @current_socket = nil
-            @metrics[:connected].set({}, 0)
+            @metrics[:write_exception].increment(server: @current_target.describe_peer, class: ex.class.to_s)
+            @metrics[:connected].set({ server: @current_target.describe_peer }, 0)
+            @logger.info("LogstashWriter") { "Error while writing to current server #{@current_target.describe_peer}: #{ex.message} (#{ex.class})" }
+            @current_target.close
+            @current_target = nil
 
             sleep INITIAL_RETRY_WAIT
           end
@@ -313,10 +313,12 @@ class LogstashWriter
 
               if next_server
                 @logger.debug("LogstashWriter") { "Trying to connect to #{next_server.to_s}" }
-                @current_socket = next_server.socket
+                @current_target = next_server
+                # Trigger a connection attempt
+                @current_target.socket
               else
                 @logger.debug("LogstashWriter") { "Could not connect to any server; pausing before trying again" }
-                @current_socket = nil
+                @current_target = nil
                 sleep retry_delay
 
                 # Calculate a longer retry delay next time we fail to connect
@@ -341,18 +343,6 @@ class LogstashWriter
     end
   end
 
-  # Generate a human-readable description of the remote end of the given
-  # socket.
-  #
-  def describe_peer(s)
-    pa = s.peeraddr
-    if pa[0] == "AF_INET6"
-      "[#{pa[3]}]:#{pa[1]}"
-    else
-      "#{pa[3]}:#{pa[1]}"
-    end
-  end
-
   # Turn the server_name given in the constructor into a list of Target
   # objects, suitable for iterating through to find someone to talk to.
   #
@@ -489,7 +479,17 @@ class LogstashWriter
     #    for any reason.
     #
     def socket
-      TCPSocket.new(@addr, @port)
+      @socket ||= TCPSocket.new(@addr, @port)
+    end
+
+    # Shut down the connection.
+    #
+    # @return [NilClass]
+    #
+    def close
+      @socket.close if @socket
+      @socket = nil
+      @describe_peer = nil
     end
 
     # Simple string representation of the target.
@@ -499,6 +499,34 @@ class LogstashWriter
     def to_s
       "#{@addr}:#{@port}"
     end
+
+    # Provide as accurate a representation of what we're *actually* connected
+    # to as we can, given the constraints of whether we're connected.
+    #
+    # To prevent unpleasantness when the other end disconnects but we still
+    # want to know who we *were* connected to, we cache the result of our
+    # cogitations.  Just in case.
+    #
+    # @return [String]
+    #
+    def describe_peer
+      @describe_peer ||= begin
+        if @socket
+          pa = @socket.peeraddr
+          if pa[0] == "AF_INET6"
+            "[#{pa[3]}]:#{pa[1]}"
+          else
+            "#{pa[3]}:#{pa[1]}"
+          end
+        else
+          nil
+        end
+      rescue Errno::ENOTCONN
+        # Peer disconnected apparently means "I forgot who I was connected
+        # to"... ¯\_(ツ)_/¯
+        nil
+      end || to_s
+    end
   end
 
   private_constant :Target

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash_writer
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Matt Palmer