logstash-input-s3-sns-sqs 1.2.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8b7f924e911ea9f2379c0b8f2415c4dbd8aa6f16
-  data.tar.gz: c9c44b0742020e1e911e01a9f5bad9bca55857ee
+  metadata.gz: fa00cb9383c9782647f404283f10281febb31622
+  data.tar.gz: a8fe32f0d3668c893ff5ef5cd253e02dee8f4fd5
 SHA512:
-  metadata.gz: 4b2f60bbe55ff50d580f7524faa78a30235ace54f8770a54baa97009aad0041d334edd5bc87822c84714142a734e86cbf859947aa6f7200f4a1c0992521a3cf3
-  data.tar.gz: d635defff1a48274967afdcd358fff1da8f9a91cde99729558f07c1b238cff866e179ab2a7122f24dc12a547d1d784d5c5736460b5afa9ed4cb76c4c8f4c3bcd
+  metadata.gz: a602a1a99073817666f83f0f0194e953c3ee6a75efd5e4e2faf4a44a1d19323133ea0c96e6fab7d306ec75f2f6384c936f2f1a134b6639e4ffe645c76af727f4
+  data.tar.gz: 8652e9ecc5c3b9342dabb842c8258adcf52f28d5184dae8f3e1bd27ffed0bac0333a79f2055be4a55bbb2e0849db8ac5426d6936fafe1c96ac25a1701038d58f

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.4.0
+- Filehandling rewritten THX to logstash-input-s3 for inspiration
+- Improve performance of gzip decoding by 10x by using Java's Zlib
+- Added multithreading via config Use: consumer_threads in config
 ## 1.2.0
 - Add codec suggestion by content-type
 - enrich metadata 

data/Gemfile

@@ -1,2 +1,11 @@
 source 'https://rubygems.org'
+
 gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/lib/logstash/inputs/s3snssqs.rb

@@ -6,6 +6,16 @@ require "logstash/timestamp"
 require "logstash/plugin_mixins/aws_config"
 require "logstash/errors"
 require 'logstash/inputs/s3sqs/patch'
+require "aws-sdk"
+require 'cgi'
+
+require 'java'
+java_import java.io.InputStream
+java_import java.io.InputStreamReader
+java_import java.io.FileInputStream
+java_import java.io.BufferedReader
+java_import java.util.zip.GZIPInputStream
+java_import java.util.zip.ZipException
 
 Aws.eager_autoload!
 
@@ -96,6 +106,10 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   config :delete_on_success, :validate => :boolean, :default => false
   # Whether the event is processed though an SNS to SQS. (S3>SNS>SQS = true |S3>SQS=false)
   config :from_sns, :validate => :boolean, :default => true
+  # To run in multiple threads use this
+  config :consumer_threads, :validate => :number, :default => 1
+  config :temporary_directory, :validate => :string, :default => File.join(Dir.tmpdir, "logstash")
+
 
   attr_reader :poller
   attr_reader :s3
@@ -106,26 +120,33 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
     require "logstash/codecs/json"
     require "logstash/codecs/json_lines"
     if content_type == "application/json_lines" then
-        @logger.info("Automatically switching from #{@codec.class.config_name} to json_lines codec", :plugin => self.class.config_name)
-        @codec = LogStash::Codecs::JSONLines.new("charset" => @codec.charset)
+      @logger.info("Automatically switching from #{@codec.class.config_name} to json_lines codec", :plugin => self.class.config_name)
+      @codec = LogStash::Codecs::JSONLines.new("charset" => @codec.charset)
     elsif content_type == "application/json" or key.end_with?(".json") then
-        @logger.info("Automatically switching from #{@codec.class.config_name} to json codec", :plugin => self.class.config_name)
-        @codec = LogStash::Codecs::JSON.new("charset" => @codec.charset)
+      @logger.info("Automatically switching from #{@codec.class.config_name} to json codec", :plugin => self.class.config_name)
+      @codec = LogStash::Codecs::JSON.new("charset" => @codec.charset)
     end
   end
 
+  public
   def register
-    require "aws-sdk"
-    require 'cgi'
+    require "fileutils"
+    require "digest/md5"
+    require "aws-sdk-resources"
+
+    @runner_threads = []
     @logger.info("Registering SQS input", :queue => @queue)
     setup_queue
+
+    FileUtils.mkdir_p(@temporary_directory) unless Dir.exist?(@temporary_directory)
   end
 
   def setup_queue
     aws_sqs_client = Aws::SQS::Client.new(aws_options_hash)
     queue_url = aws_sqs_client.get_queue_url({ queue_name: @queue, queue_owner_aws_account_id: @queue_owner_aws_account_id})[:queue_url]
     @poller = Aws::SQS::QueuePoller.new(queue_url, :client => aws_sqs_client)
-    @s3 = Aws::S3::Client.new(aws_options_hash)
+    @s3_client = Aws::S3::Client.new(aws_options_hash)
+    @s3_resource = get_s3object
   rescue Aws::SQS::Errors::ServiceError => e
     @logger.error("Cannot establish connection to Amazon SQS", :error => e)
     raise LogStash::ConfigurationError, "Verify the SQS queue name and your credentials"
@@ -133,17 +154,17 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
 
   def polling_options
     {
-      # we will query 1 message at a time, so we can ensure correct error handling if we can't download a single file correctly
-      # (we will throw :skip_delete if download size isn't correct to process the event again later
-      # -> set a reasonable "Default Visibility Timeout" for your queue, so that there's enough time to process the log files)
-      :max_number_of_messages => 1,
-      # we will use the queue's setting, a good value is 10 seconds
-      # (to ensure fast logstash shutdown on the one hand and few api calls on the other hand)
-      :wait_time_seconds => nil,
+        # we will query 1 message at a time, so we can ensure correct error handling if we can't download a single file correctly
+        # (we will throw :skip_delete if download size isn't correct to process the event again later
+        # -> set a reasonable "Default Visibility Timeout" for your queue, so that there's enough time to process the log files)
+        :max_number_of_messages => 1,
+        # we will use the queue's setting, a good value is 10 seconds
+        # (to ensure fast logstash shutdown on the one hand and few api calls on the other hand)
+        :wait_time_seconds => nil,
     }
   end
 
-  def handle_message(message, queue)
+  def handle_message(message, queue, instance_codec)
     hash = JSON.parse message.body
     @logger.debug("handle_message", :hash => hash, :message => message)
     #If send via sns there is an additional JSON layer
@@ -152,97 +173,239 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
     end
     # there may be test events sent from the s3 bucket which won't contain a Records array,
     # we will skip those events and remove them from queue
-      if hash['Records'] then
-	      # typically there will be only 1 record per event, but since it is an array we will
-        # treat it as if there could be more records
-        hash['Records'].each do |record|
-          @logger.debug("We found a record", :record => record)
-	        # in case there are any events with Records that aren't s3 object-created events and can't therefore be
-          # processed by this plugin, we will skip them and remove them from queue
-          if record['eventSource'] == EVENT_SOURCE and record['eventName'].start_with?(EVENT_TYPE) then
-            @logger.debug("It is a valid record")
-	          bucket = CGI.unescape(record['s3']['bucket']['name'])
-	          key    = CGI.unescape(record['s3']['object']['key'])
-
-
-            # try download and :skip_delete if it fails
-            begin
-              response = @s3.get_object(
-                bucket: bucket,
-                key: key,
-              )
-            rescue => e
-              @logger.warn("issuing :skip_delete on failed download", :bucket => bucket, :object => key, :error => e)
-              throw :skip_delete
-            end
-
-            # verify downloaded content size
-            if response.content_length == record['s3']['object']['size'] then
-              body = response.body
-              # if necessary unzip
-              if response.content_encoding == "gzip" or record['s3']['object']['key'].end_with?(".gz") then
-                @logger.debug("Ohhh i´ll try to unzip")
-	              begin
-		              temp = Zlib::GzipReader.new(body)
-                  rescue => e
-                  @logger.warn("content is marked to be gzipped but can't unzip it, assuming plain text", :bucket => bucket, :object => key, :error => e)
-                  temp = body
-                end
-                body = temp
-              end
-              # Make a suggestion for a good codec
-              suggest_codec(response.content_type,record['s3']['object']['key'])
-              # process the plain text content
-              begin
-                lines = body.read.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370").split(/\n/)
-                lines.each do |line|
-                  @logger.debug("Decorating the event")
-		              @codec.decode(line) do |event|
-                  decorate(event)
-
-                  event.set('[@metadata][s3_bucket_name]', record['s3']['bucket']['name'])
-                  event.set('[@metadata][s3_object_key]', record['s3']['object']['key'])
-                  if match=/#{s3_key_prefix}\/?(?<type_folder>.*?)\/.*/.match(key)
-                    event.set('[@metadata][s3_object_folder]', match['type_folder'])
-                  end
-                  queue << event
-                  end
-                end
-              rescue => e
-                @logger.warn("issuing :skip_delete on failed plain text processing", :bucket => bucket, :object => key, :error => e)
-                throw :skip_delete
-              end
-
-              # Delete the files from S3
-              begin
-                @s3.delete_object(bucket: bucket, key: key) if @delete_on_success
-              rescue => e
-                @logger.warn("Failed to delete S3 object", :bucket => bucket, :object => key, :error => e)
-              end
-            # otherwise try again later
-            else
-              @logger.warn("issuing :skip_delete on wrong download content size", :bucket => bucket, :object => key,
-                :download_size => response.content_length, :expected => record['s3']['object']['size'])
-              throw :skip_delete
-            end
+    if hash['Records'] then
+      # typically there will be only 1 record per event, but since it is an array we will
+      # treat it as if there could be more records
+      hash['Records'].each do |record|
+        @logger.debug("We found a record", :record => record)
+        # in case there are any events with Records that aren't s3 object-created events and can't therefore be
+        # processed by this plugin, we will skip them and remove them from queue
+        if record['eventSource'] == EVENT_SOURCE and record['eventName'].start_with?(EVENT_TYPE) then
+          @logger.debug("It is a valid record")
+          bucket = CGI.unescape(record['s3']['bucket']['name'])
+          key    = CGI.unescape(record['s3']['object']['key'])
+
+          # try download and :skip_delete if it fails
+          #if record['s3']['object']['size'] < 10000000 then
+          process_log(bucket, key, instance_codec, queue)
+          #else
+          #  @logger.info("Your file is too big")
+          #end
+        end
+      end
+    end
+  end
+
+  private
+  def process_log(bucket , key, instance_codec, queue)
+    s3bucket = @s3_resource.bucket(bucket)
+    @logger.debug("Lets go reading file", :bucket => bucket, :key => key)
+    object = s3bucket.object(key)
+    filename = File.join(temporary_directory, File.basename(key))
+    if download_remote_file(object, filename)
+      if process_local_log( filename, key, instance_codec, queue)
+        delete_file_from_bucket(object)
+        FileUtils.remove_entry_secure(filename, true)
+      end
+    else
+      FileUtils.remove_entry_secure(filename, true)
+    end
+  end
+
+  private
+  # Stream the remove file to the local disk
+  #
+  # @param [S3Object] Reference to the remove S3 objec to download
+  # @param [String] The Temporary filename to stream to.
+  # @return [Boolean] True if the file was completely downloaded
+  def download_remote_file(remote_object, local_filename)
+    completed = false
+    @logger.debug("S3 input: Download remote file", :remote_key => remote_object.key, :local_filename => local_filename)
+    File.open(local_filename, 'wb') do |s3file|
+      return completed if stop?
+      remote_object.get(:response_target => s3file)
+    end
+    completed = true
+
+    return completed
+  end
+
+  private
+
+  # Read the content of the local file
+  #
+  # @param [Queue] Where to push the event
+  # @param [String] Which file to read from
+  # @return [Boolean] True if the file was completely read, false otherwise.
+  def process_local_log(filename, key, instance_codec, queue)
+    @logger.debug('Processing file', :filename => filename)
+    metadata = {}
+    i=1
+    # Currently codecs operates on bytes instead of stream.
+    # So all IO stuff: decompression, reading need to be done in the actual
+    # input and send as bytes to the codecs.
+    read_file(filename) do |line|
+      if stop?
+        @logger.warn("Logstash S3 input, stop reading in the middle of the file, we will read it again when logstash is started")
+        return false
+      end
+      #@logger.info("read line #{i}", :line => line)
+      #line = line.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370")
+      instance_codec.decode(line) do |event|
+        #@logger.info("decorate event")
+        # We are making an assumption concerning cloudfront
+        # log format, the user will use the plain or the line codec
+        # and the message key will represent the actual line content.
+        # If the event is only metadata the event will be drop.
+        # This was the behavior of the pre 1.5 plugin.
+        #
+        # The line need to go through the codecs to replace
+        # unknown bytes in the log stream before doing a regexp match or
+        # you will get a `Error: invalid byte sequence in UTF-8'
+        #event = LogStash::Event.new("message" => @message)
+        if event_is_metadata?(event)
+          @logger.debug('Event is metadata, updating the current cloudfront metadata', :event => event)
+          update_metadata(metadata, event)
+        else
+
+          decorate(event)
+
+          event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
+          event.set("cloudfront_fields", metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
+
+          event.set("[@metadata][s3]", { "key" => key })
+
+          if match=/#{s3_key_prefix}\/?(?<type_folder>.*?)\/.*/.match(key)
+            event.set('[@metadata][s3_object_folder]', match['type_folder'])
           end
+          #@logger.info("queue event #{i}")
+          #i += 1
+          queue << event
         end
       end
+    end
+    #@logger.info("event pre flush", :event => event)
+    # #ensure any stateful codecs (such as multi-line ) are flushed to the queue
+    instance_codec.flush do |event|
+      queue << event
+    end
+
+    return true
+  end # def process_local_log
+
+  private
+  def read_file(filename, &block)
+    if gzip?(filename)
+      read_gzip_file(filename, block)
+    else
+      read_plain_file(filename, block)
+    end
+  end
+
+  def read_plain_file(filename, block)
+    File.open(filename, 'rb') do |file|
+      file.each(&block)
+    end
+  end
+
+  private
+  def read_gzip_file(filename, block)
+    file_stream = FileInputStream.new(filename)
+    gzip_stream = GZIPInputStream.new(file_stream)
+    decoder = InputStreamReader.new(gzip_stream, "UTF-8")
+    buffered = BufferedReader.new(decoder)
+
+    while (line = buffered.readLine())
+      block.call(line)
+    end
+  rescue ZipException => e
+    @logger.error("Gzip codec: We cannot uncompress the gzip file", :filename => filename)
+    raise e
+  ensure
+    buffered.close unless buffered.nil?
+    decoder.close unless decoder.nil?
+    gzip_stream.close unless gzip_stream.nil?
+    file_stream.close unless file_stream.nil?
+  end
+
+  private
+  def gzip?(filename)
+    filename.end_with?('.gz','.gzip')
   end
 
+
+  private
+  def delete_file_from_bucket(object)
+    if @delete_on_success
+      object.delete()
+    end
+  end
+
+  private
+  def get_s3object
+    s3 = Aws::S3::Resource.new(client: @s3_client)
+  end
+
+  private
+  def event_is_metadata?(event)
+    return false unless event.get("message").class == String
+    line = event.get("message")
+    version_metadata?(line) || fields_metadata?(line)
+  end
+
+  private
+  def version_metadata?(line)
+    line.start_with?('#Version: ')
+  end
+
+  private
+  def fields_metadata?(line)
+    line.start_with?('#Fields: ')
+  end
+
+  private
+  def update_metadata(metadata, event)
+    line = event.get('message').strip
+
+    if version_metadata?(line)
+      metadata[:cloudfront_version] = line.split(/#Version: (.+)/).last
+    end
+
+    if fields_metadata?(line)
+      metadata[:cloudfront_fields] = line.split(/#Fields: (.+)/).last
+    end
+  end
+
+  public
   def run(queue)
     # ensure we can stop logstash correctly
-    poller.before_request do |stats|
-      if stop? then
-        @logger.warn("issuing :stop_polling on stop?", :queue => @queue)
-        # this can take up to "Receive Message Wait Time" (of the sqs queue) seconds to be recognized
-        throw :stop_polling
-      end
-    end
-    # poll a message and process it
-    run_with_backoff do
-      poller.poll(polling_options) do |message|
-        handle_message(message, queue)
+    @runner_threads = consumer_threads.times.map { |consumer| thread_runner(queue) }
+    @runner_threads.each { |t| t.join }
+  end
+
+  public
+  def stop
+    @runner_threads.each { |c| c.wakeup }
+  end
+
+  private
+  def thread_runner(queue)
+    Thread.new do
+      @logger.info("Starting new thread")
+      begin
+        poller.before_request do |stats|
+          if stop? then
+            @logger.warn("issuing :stop_polling on stop?", :queue => @queue)
+            # this can take up to "Receive Message Wait Time" (of the sqs queue) seconds to be recognized
+            throw :stop_polling
+          end
+        end
+        # poll a message and process it
+        run_with_backoff do
+          poller.poll(polling_options) do |message|
+            handle_message(message, queue, @codec.clone)
+          end
+        end
       end
     end
   end
@@ -266,5 +429,4 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
       retry
     end
   end
-
-end # class
+end # class

data/logstash-input-s3-sns-sqs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-s3-sns-sqs'
-  s.version         = '1.2.0'
+  s.version         = '1.4.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -19,11 +19,12 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 2.1.12", "<= 2.99"
 
   s.add_runtime_dependency 'logstash-codec-json'
-  s.add_runtime_dependency "logstash-mixin-aws", ">= 1.0.0"
+  s.add_runtime_dependency "logstash-mixin-aws"
 
   s.add_development_dependency 'logstash-devutils'
+
 end
 

metadata

@@ -1,21 +1,21 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3-sns-sqs
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.4.0
 platform: ruby
 authors:
 - Christian Herweg
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-02-19 00:00:00.000000000 Z
+date: 2018-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '1.60'
+        version: 2.1.12
     - - <=
       - !ruby/object:Gem::Version
         version: '2.99'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '1.60'
+        version: 2.1.12
     - - <=
       - !ruby/object:Gem::Version
         version: '2.99'
@@ -49,7 +49,7 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '0'
   name: logstash-mixin-aws
   prerelease: false
   type: :runtime
@@ -57,7 +57,7 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: