franz 2.1.6 → 2.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 192d0de743aeb1560eb55145d11b06c2f1f5ba61
-  data.tar.gz: 145887b6c3313ad29f4eb88f9d17df62899a27c2
+  metadata.gz: b6f56ba47d6adce9b725e89c82f9d77763b87162
+  data.tar.gz: 923a2838012eb28fec495490887326786fbfbbb8
 SHA512:
-  metadata.gz: b2a4cde46ea536888c90c4f2fb2611a7f4526840c2a422d85f99b646385b580e48f1eee92fc91c0422c1287d075d8e92530c80f060e0bdeb65cbace48276517f
-  data.tar.gz: fdbb262b5620837f14d92986e12d946decd4b335665953ff7784151dcf6fafee8ba0843cbe14179d9a9e141cdbb2fdabe8ce187043ebc02db3d0ef0ac21787b0
+  metadata.gz: dd5ce1689ecdd4667476b11ff74540bc4f262fec7959c4b9c6b1d488809e3f33e1277e51b05d4aa33ef4c851e364d51dab9c05d1478c966c0b9c22033234486b
+  data.tar.gz: 9985397b685ae376718dca6ba2802367ec1c7dc8d8b856e777bd630f0f4053681713d067d783af9b5f486e53ce6692612ce91df576f3e69e26d6f2c38340d342

data/Readme.md

@@ -232,6 +232,7 @@ for Chef.
 - Running Franz without arguments now prints help text (i.e. `--config` is now required)
 - Rename `StdOut` to `Device`, no longer experimental
 - Allow single state file if no glob is used
+- Add autoreconnect logic to `Kafka` output
 - Handle `EACCESS` on `IO.read` gracefully
 - New `HTTP` output (experimental)
 - Remove default line limit

data/VERSION

@@ -1 +1 @@
-2.1.6
+2.1.7

data/lib/franz/output/http.rb

@@ -118,7 +118,7 @@ module Franz
         request.body = body
         @http.request(request)
       rescue EOFError, Errno::ECONNREFUSED, Errno::EPIPE
-        log.warn event: 'output reconnect'
+        log.warn event: 'output dropped'
         open_uri
         sleep 1
         retry

data/lib/franz/output/kafka.rb

@@ -52,30 +52,25 @@ module Franz
         @flush_interval = opts[:output].delete :flush_interval
         @topic = opts[:output].delete :topic
 
-        kafka_brokers = opts[:output].delete(:brokers) || %w[ localhost:9092 ]
-        kafka_client_id = opts[:output].delete :client_id
-        kafka_config = opts[:output].map { |k,v|
+        @kafka_brokers = opts[:output].delete(:brokers) || %w[ localhost:9092 ]
+        @kafka_client_id = opts[:output].delete :client_id
+        @kafka_config = opts[:output].map { |k,v|
           [ k, v.is_a?(String) ? v.to_sym : v ]
         }
 
-        @kafka = Poseidon::Producer.new \
-          kafka_brokers,
-          kafka_client_id,
-          Hash[kafka_config]
+        kafka_connect
 
         @lock = Mutex.new
         @messages = []
 
 
         @thread = Thread.new do
-          loop do
+          until @stop
             @lock.synchronize do
-              ready_messages = @messages
-              @messages = []
-              @kafka.send_messages ready_messages unless ready_messages.empty?
+              num_messages = kafka_send @messages
               log.debug \
                 event: 'periodic flush',
-                num_messages: ready_messages.size
+                num_messages: num_messages
             end
 
             sleep @flush_interval
@@ -96,14 +91,11 @@ module Franz
             @lock.synchronize do
               @messages << Poseidon::MessageToSend.new(@topic, payload)
 
-              @statz.inc :num_output
-
               if @messages.size >= @flush_size
-                @kafka.send_messages @messages
+                num_messages = kafka_send @messages
                 log.debug \
                   event: 'flush',
-                  num_messages: @messages.size
-                @messages = []
+                  num_messages: num_messages
               end
             end
 
@@ -136,6 +128,27 @@ module Franz
     private
       def log ; @logger end
 
+      def kafka_connect
+        @kafka = Poseidon::Producer.new \
+          @kafka_brokers,
+          @kafka_client_id,
+          Hash[@kafka_config]
+      end
+
+      def kafka_send messages
+        return 0 if @messages.empty?
+        @kafka.send_messages @messages
+        @statz.inc :num_output, @messages.length
+        size = @messages.size
+        @messages = []
+        return size
+      rescue Poseidon::Errors::UnableToFetchMetadata
+        log.warn event: 'output dropped'
+        kafka_connect
+        sleep 1
+        retry
+      end
+
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: franz
 version: !ruby/object:Gem::Version
-  version: 2.1.6
+  version: 2.1.7
 platform: ruby
 authors:
 - Sean Clemmer