logstash-input-s3-sns-sqs 1.4.0 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fa00cb9383c9782647f404283f10281febb31622
-  data.tar.gz: a8fe32f0d3668c893ff5ef5cd253e02dee8f4fd5
+  metadata.gz: b252a2e8b2c9ef1fa911f4f3032508ddfda53411
+  data.tar.gz: 10b201c1f0e1a2a5858682efa109c99da3d0d3bf
 SHA512:
-  metadata.gz: a602a1a99073817666f83f0f0194e953c3ee6a75efd5e4e2faf4a44a1d19323133ea0c96e6fab7d306ec75f2f6384c936f2f1a134b6639e4ffe645c76af727f4
-  data.tar.gz: 8652e9ecc5c3b9342dabb842c8258adcf52f28d5184dae8f3e1bd27ffed0bac0333a79f2055be4a55bbb2e0849db8ac5426d6936fafe1c96ac25a1701038d58f
+  metadata.gz: 75f71948df7e26e7ac607219532cdb1a8df156ada93dab895560fbcf215b06308e1e900cc6c21450d395a9ec5d48a841e6c3104fd244e380074e2337088d6d5a
+  data.tar.gz: ad9ef3e7094750e8976cd6b02ebfb9cbedbf4ac15ee6585b91c76d9de237436bf8e3a684cbed336db029e0c0c1a748b6989a12af73e4bce2487e0194699b0159

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.4.1
+- Fix: Last event in file not decorated
+- Adjust metadata namings
+- Event decoration in private method now.
 ## 1.4.0
 - Filehandling rewritten THX to logstash-input-s3 for inspiration
 - Improve performance of gzip decoding by 10x by using Java's Zlib

data/lib/logstash/inputs/s3snssqs.rb

@@ -98,6 +98,9 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
 
   default :codec, "plain"
 
+  # The message string to use in the event.
+  config :message, :validate => :string, :default => "Hello World!"
+
   # Name of the SQS Queue to pull messages from. Note that this is just the name of the queue, not the URL or ARN.
   config :queue, :validate => :string, :required => true
   config :s3_key_prefix, :validate => :string, :default => ''
@@ -203,7 +206,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
     object = s3bucket.object(key)
     filename = File.join(temporary_directory, File.basename(key))
     if download_remote_file(object, filename)
-      if process_local_log( filename, key, instance_codec, queue)
+      if process_local_log( filename, key, instance_codec, queue, bucket)
         delete_file_from_bucket(object)
         FileUtils.remove_entry_secure(filename, true)
       end
@@ -237,10 +240,9 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   # @param [Queue] Where to push the event
   # @param [String] Which file to read from
   # @return [Boolean] True if the file was completely read, false otherwise.
-  def process_local_log(filename, key, instance_codec, queue)
+  def process_local_log(filename, key, instance_codec, queue, bucket)
     @logger.debug('Processing file', :filename => filename)
     metadata = {}
-    i=1
     # Currently codecs operates on bytes instead of stream.
     # So all IO stuff: decompression, reading need to be done in the actual
     # input and send as bytes to the codecs.
@@ -262,37 +264,42 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
         # The line need to go through the codecs to replace
         # unknown bytes in the log stream before doing a regexp match or
         # you will get a `Error: invalid byte sequence in UTF-8'
-        #event = LogStash::Event.new("message" => @message)
-        if event_is_metadata?(event)
-          @logger.debug('Event is metadata, updating the current cloudfront metadata', :event => event)
-          update_metadata(metadata, event)
-        else
-
-          decorate(event)
-
-          event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
-          event.set("cloudfront_fields", metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
-
-          event.set("[@metadata][s3]", { "key" => key })
-
-          if match=/#{s3_key_prefix}\/?(?<type_folder>.*?)\/.*/.match(key)
-            event.set('[@metadata][s3_object_folder]', match['type_folder'])
-          end
-          #@logger.info("queue event #{i}")
-          #i += 1
-          queue << event
-        end
+        local_decorate(event, key, metadata, bucket)
+        queue << event
       end
     end
+    @logger.debug("end if file #{filename}")
     #@logger.info("event pre flush", :event => event)
     # #ensure any stateful codecs (such as multi-line ) are flushed to the queue
     instance_codec.flush do |event|
+      local_decorate(event, key, metadata, bucket)
+      @logger.debug("We´e to flush an incomplete event...", :event => event)
       queue << event
     end
 
     return true
   end # def process_local_log
 
+  private
+  def local_decorate(event, key, metadata, bucket)
+    if event_is_metadata?(event)
+      @logger.debug('Event is metadata, updating the current cloudfront metadata', :event => event)
+      update_metadata(metadata, event)
+    else
+
+      decorate(event)
+
+      event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
+      event.set("cloudfront_fields", metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
+
+      event.set("[@metadata][s3]", { "object_key" => key })
+      event.set("[@metadata][s3]", { "bucket_name" => bucket })
+      if match=/#{s3_key_prefix}\/?(?<type_folder>.*?)\/.*/.match(key)
+        event.set('[@metadata][s3]', {"s3_object_folder" => match['type_folder']})
+      end
+    end
+  end
+
   private
   def read_file(filename, &block)
     if gzip?(filename)

data/logstash-input-s3-sns-sqs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-s3-sns-sqs'
-  s.version         = '1.4.0'
+  s.version         = '1.4.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3-sns-sqs
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.4.1
 platform: ruby
 authors:
 - Christian Herweg
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-03-16 00:00:00.000000000 Z
+date: 2018-03-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement