logstash-output-sumologic 1.4.0 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ab9e1ae438a4d6030629b00a67c0536aba6995f6a2bae1f952b9737cf2c28189
-  data.tar.gz: fdfec75b2d7c4bcd5c242c1d142d041648d450e3dfcb7a576de9531986f5438d
+  metadata.gz: a93e4aba1af87a7e0c3ea3570e5ff177e4307736dacf2fde1da78dbe4150a2da
+  data.tar.gz: 9ceb227f390a49467f7195daa96152335922275b714dda87215f11c16273d060
 SHA512:
-  metadata.gz: 66620931f50b2c241835824e26e16e0c05ef8c4ec9ac5ad3770d579ae345ef4486c68de23de8c7c284e3005598231738f75bef59704977a65e3911b992907951
-  data.tar.gz: 294a5910f5ec91ace98405439411d26ddf1e3c858e6d17cefb67d8b2b3c0b534c101cdd93faeab08c34dd42f77f5fb078e03b853e6937d63b1ed9f56ba69b840
+  metadata.gz: 88b55069b26026545a9cbb755555fa53e0570b120ea37ded4ae487da989f0054a6fb1259998a8e9eb038c0314a4450afa70fe658d372030f4eae0f25e7247b5b
+  data.tar.gz: 6ca3a09f3f297c87c06707dd6a3b74d8210cb46ac60dde3f2663b2509cd87274d92d67d49203194a4def5e70dd314190a4887ac9401199889ecbaf79b22bb77a

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Change Log
 
+## 1.4.1 (2022-03-09)
+
+- [#73](https://github.com/SumoLogic/logstash-output-sumologic/pull/73) fix: remove deadlock possibility by adding resend_queue queue
+
 ## 1.4.0 (2021-09-27)
 
 - [#68](https://github.com/SumoLogic/logstash-output-sumologic/pull/68) feat: retry on 502 error code

data/lib/logstash/outputs/sumologic/message_queue.rb

@@ -27,8 +27,8 @@ module LogStash; module Outputs; class SumoLogic;
         end
     end # def enq
 
-    def deq()
-      batch = @queue.deq()
+    def deq(non_block: false)
+      batch = @queue.deq(non_block: non_block)
       batch_size = batch.payload.bytesize
       @stats.record_deque(batch_size)
       @queue_bytesize.update { |v| v - batch_size }

data/lib/logstash/outputs/sumologic/sender.rb

@@ -24,10 +24,14 @@ module LogStash; module Outputs; class SumoLogic;
       @sender_max = (config["sender_max"] ||= 1) < 1 ? 1 : config["sender_max"]
       @sleep_before_requeue = config["sleep_before_requeue"] ||= 30
       @stats_enabled = config["stats_enabled"] ||= false
+      @iteration_sleep = 0.3
 
       @tokens = SizedQueue.new(@sender_max)
       @sender_max.times { |t| @tokens << t }
 
+      # Make resend_queue twice as big as sender_max,
+      # because if one batch is processed, the next one is already waiting in the thread
+      @resend_queue = SizedQueue.new(2*@sender_max)
       @compressor = LogStash::Outputs::SumoLogic::Compressor.new(config)
 
     end # def initialize
@@ -39,9 +43,24 @@ module LogStash; module Outputs; class SumoLogic;
       @stopping.make_false()
       @sender_t = Thread.new {
         while @stopping.false?
-          batch = @queue.deq()
+          begin
+            # Resend batch if any in the queue
+            batch = @resend_queue.deq(non_block: true)
+          rescue ThreadError
+            # send new batch otherwise
+            begin
+              batch = @queue.deq(non_block: true)
+            rescue ThreadError
+              Stud.stoppable_sleep(@iteration_sleep) { @stopping.true? }
+              next
+            end
+          end
           send_request(batch)
         end # while
+        @resend_queue.size.times.map { |queue|
+          batch = queue.deq()
+          send_request(batch)
+        }
         @queue.drain().map { |batch| 
           send_request(batch)
         }
@@ -98,6 +117,7 @@ module LogStash; module Outputs; class SumoLogic;
         return
       end
       
+      # wait for token so we do not exceed number of request in background
       token = @tokens.pop()
 
       if @stats_enabled && content.start_with?(STATS_TAG)
@@ -111,11 +131,9 @@ module LogStash; module Outputs; class SumoLogic;
         :content_size => content.size,
         :content => content[0..20],
         :payload_size => body.size)
+
+      # send request in background
       request = @client.send(:background).send(:post, @url, :body => body, :headers => headers)
-      
-      request.on_complete do
-        @tokens << token
-      end
   
       request.on_success do |response|
         @stats.record_response_success(response.code)
@@ -126,12 +144,16 @@ module LogStash; module Outputs; class SumoLogic;
             :headers => headers,
             :contet => content[0..20])
           if response.code == 429 || response.code == 502 || response.code == 503 || response.code == 504
+            # requeue and release token
             requeue_message(batch)
+            @tokens << token
           end
         else
           log_dbg("request accepted",
             :token => token,
             :code => response.code)
+          # release token
+          @tokens << token
         end
       end
   
@@ -143,6 +165,8 @@ module LogStash; module Outputs; class SumoLogic;
           :class => exception.class.name,
           :backtrace => exception.backtrace)
         requeue_message(batch)
+        # requeue and release token
+        @tokens << token
       end      
 
       @stats.record_request(content.bytesize, body.bytesize)
@@ -162,7 +186,7 @@ module LogStash; module Outputs; class SumoLogic;
           :content => content[0..20],
           :headers => batch.headers)
         Stud.stoppable_sleep(@sleep_before_requeue) { @stopping.true? }
-        @queue.enq(batch)
+        @resend_queue.enq(batch)
       end
     end # def reque_message
 

data/logstash-output-sumologic.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-sumologic'
-  s.version       = '1.4.0'
+  s.version       = '1.4.1'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Deliever the log to Sumo Logic cloud service.'
   s.description   = 'This gem is a Logstash output plugin to deliver the log or metrics to Sumo Logic cloud service. Go to https://github.com/SumoLogic/logstash-output-sumologic for getting help, reporting issues, etc.'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-sumologic
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.4.1
 platform: ruby
 authors:
 - Sumo Logic
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-27 00:00:00.000000000 Z
+date: 2022-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: manticore