logstash-input-s3-sns-sqs 2.0.0 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 253c85cd1d1dfa22a59d282a0eeae4e5a71c5630473db65a768fe3e00131adc9
-  data.tar.gz: 5e98e0d9b47c7f9b47d6e11aefa2d5c14f59e4fe6c66a7bd0934a250bb8fbcfb
+  metadata.gz: 9989a06ffe035de2692ed0331fc45b9da60ddab3270826487eeb75fd20e882cb
+  data.tar.gz: e36504f3dca4413e4b34edee8e1a82203444837f785a36b8e2450f83f98102e3
 SHA512:
-  metadata.gz: ee38fcc3de70af94b7de1570b054cdf435224c77b64bebd1db9b8eee3a4097b91f0320ce09d732fdefd8b1d8e0ca722c0a9b799e49dceda881e36bb7b26417e0
-  data.tar.gz: d6818a6bdead5aae583a09e2af6e1e869a7fee4902c6503def1150584afb16f415a825eb2a34b2e8a05130d8e6348bbda28f4885852e8eb92ec552e68382d15c
+  metadata.gz: e538a1426547653719bf0f2dba0aaae03bb80c50b5259ddd69871c209961cc21c389ba587028aa1bc57b017fa4445929350a7cab5833d713d5a0e8d4289f530b
+  data.tar.gz: 589100a9362b9bdb79df9c5ad8e68568175bc8384c6c5302634a2923b82edd8ce4ce6455bb269089e4aed314844bf2ea2cabe39d55f6d3d38c45fa8448b2fa77

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+##2.0.1
+FIX:
+- Deadlock while message decoding
+- make method stop? public
+
 ##2.0.0
 Breaking Changes:
 - s3_key_prefix was never functional and will be removed. Actually only used for metadata.folder backward compatibility.

data/lib/logstash/inputs/codec_factory.rb

@@ -17,7 +17,7 @@ class CodecFactory
       @codecs[codec] = get_codec_plugin(codec)
     end
     @logger.debug("Switching to codec #{codec}") if codec != 'default'
-    return @codecs[codec]
+    return @codecs[codec].clone
   end
 
   private

data/lib/logstash/inputs/s3/downloader.rb

@@ -15,12 +15,14 @@ class S3Downloader
     # (from docs) WARNING:
     # yielding data to a block disables retries of networking errors!
     begin
+      #@logger.info("Download File", :file => record)
       @factory.get_s3_client(record[:bucket]) do |s3|
         response = s3.get_object(
           bucket: record[:bucket],
           key: record[:key],
           response_target: record[:local_file]
         )
+        #@logger.info("READY: File", :file => record, :response => response)
       end
     rescue Aws::S3::Errors::ServiceError => e
       @logger.error("Unable to download file. Requeuing the message", :error => e, :record => record)
@@ -32,6 +34,7 @@ class S3Downloader
   end
 
   def cleanup_local_object(record)
+    #@logger.info("Cleaning up file", :file => record[:local_file])
     FileUtils.remove_entry_secure(record[:local_file], true) if ::File.exists?(record[:local_file])
   rescue Exception => e
     @logger.warn("Could not delete file", :file => record[:local_file], :error => e)

data/lib/logstash/inputs/s3snssqs.rb

@@ -7,11 +7,13 @@ require "logstash/shutdown_watcher"
 require "logstash/errors"
 require 'logstash/inputs/s3sqs/patch'
 require "aws-sdk"
+
 # "object-oriented interfaces on top of API clients"...
 # => Overhead. FIXME: needed?
 #require "aws-sdk-resources"
 require "fileutils"
 require "concurrent"
+require 'tmpdir'
 # unused in code:
 #require "stud/interval"
 #require "digest/md5"
@@ -265,8 +267,10 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   def run(logstash_event_queue)
     #LogStash::ShutdownWatcher.abort_threshold(30)
     # start them
-    @worker_threads = @consumer_threads.times.map do |_|
-      run_worker_thread(logstash_event_queue)
+    @queue_mutex = Mutex.new
+    #@consumer_threads= 1
+    @worker_threads = @consumer_threads.times.map do |thread_id|
+      run_worker_thread(logstash_event_queue, thread_id)
     end
     # and wait (possibly infinitely) for them to shut down
     @worker_threads.each { |t| t.join }
@@ -286,18 +290,24 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
     end
   end
 
+  def stop?
+    @received_stop.value
+  end
+
   # --- END plugin interface ------------------------------------------#
 
   private
 
-  def run_worker_thread(queue)
+  def run_worker_thread(queue, thread_id)
     Thread.new do
       @logger.info("Starting new worker thread")
+      temporary_directory = Dir.mktmpdir("#{@temporary_directory}/")
       @sqs_poller.run do |record|
         throw :skip_delete if stop?
         @logger.debug("Outside Poller: got a record", :record => record)
         # record is a valid object with the keys ":bucket", ":key", ":size"
-        record[:local_file] = File.join(@temporary_directory, File.basename(record[:key]))
+        record[:local_file] = File.join(temporary_directory, File.basename(record[:key]))
+        LogStash::Util.set_thread_name("[Processor #{thread_id} -  Working on: #{record[:key]}")
         if @s3_downloader.copy_s3object_to_disk(record)
           completed = catch(:skip_delete) do
             process(record, queue)
@@ -334,9 +344,4 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
     # return input hash (convenience)
     return myhash
   end
-
-  def stop?
-    @received_stop.value
-  end
-
 end # class

data/lib/logstash/inputs/s3snssqs/log_processor.rb

@@ -16,17 +16,27 @@ module LogProcessor
     folder = record[:folder]
     type = @type_by_folder[folder] #if @type_by_folder.key?(folder)
     metadata = {}
+    #@logger.info("processing file",:thread => Thread.current[:name], :local_file => record[:local_file])
+    line_count = 0
+    event_count = 0
     read_file(file) do |line|
+      line_count += 1
+      #@logger.info("got a yielded line", :line_count => line_count) if line_count < 10
       if stop?
         @logger.warn("Abort reading in the middle of the file, we will read it again when logstash is started")
         throw :skip_delete
       end
       line = line.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370")
+      #@logger.info("ENcoded line", :line_count => line_count) if line_count < 10
       codec.decode(line) do |event|
         decorate_event(event, metadata, type, record[:key], record[:bucket], folder)
+        event_count += 1
         logstash_event_queue << event
+        #@logger.info("queued event ", :lines => line_count, :events => event_count, :thread => Thread.current[:name]) if event_count < 10
       end
+      #@logger.info("DEcoded line", :line_count => line_count) if line_count < 10
     end
+    #@logger.info("queued all events ", :lines => line_count, :events => event_count, :thread => Thread.current[:name])
     # ensure any stateful codecs (such as multi-line ) are flushed to the queue
     codec.flush do |event|
       decorate_event(event, metadata, type, record[:key], record[:bucket], folder)
@@ -65,7 +75,8 @@ module LogProcessor
     @logger.warn("Problem while gzip detection", :error => e)
   end
 
-  def read_file(filename)
+  def read_file(filename, &block)
+    #@logger.info("begin read_file",:thread => Thread.current[:name])
     completed = false
     zipped = gzip?(filename)
     file_stream = FileInputStream.new(filename)
@@ -76,19 +87,26 @@ module LogProcessor
       decoder = InputStreamReader.new(file_stream, 'UTF-8')
     end
     buffered = BufferedReader.new(decoder)
-
-    while (line = buffered.readLine())
-      yield(line)
+    line = buffered.readLine()
+    #@logger.info("read first line", :line => line)
+    while (!line.nil?)
+      block.call(line)
+      line = buffered.readLine()
+      #@logger.info("next line read",:line => line)
     end
+    #@logger.info("finished read_file",:thread => Thread.current[:name])
     completed = true
+
   rescue ZipException => e
     @logger.error("Gzip codec: We cannot uncompress the gzip file", :filename => filename, :error => e)
+    return nil
   ensure
     buffered.close unless buffered.nil?
     decoder.close unless decoder.nil?
     gzip_stream.close unless gzip_stream.nil?
     file_stream.close unless file_stream.nil?
     throw :skip_delete unless completed
+    return nil
   end
 
   def event_is_metadata?(event)

data/lib/logstash/inputs/sqs/poller.rb

@@ -90,7 +90,7 @@ class SqsPoller
         failed = false
         begin
           preprocess(message) do |record|
-            @logger.debug("we got a record", :record => record)
+            #@logger.info("we got a record", :record => record)
             yield(record) #unless record.nil? - unnecessary; implicit
           end
         rescue Exception => e
@@ -100,6 +100,7 @@ class SqsPoller
         end
         # at this time the extender has either fired or is obsolete
         extender.kill
+        #@logger.info("Inside Poller: killed background thread", :message => message)
         extender = nil
         throw :skip_delete if failed
       end

data/logstash-input-s3-sns-sqs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-s3-sns-sqs'
-  s.version         = '2.0.0'
+  s.version         = '2.0.1'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3-sns-sqs
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.1
 platform: ruby
 authors:
 - Christian Herweg
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-06-24 00:00:00.000000000 Z
+date: 2019-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement