logstash-input-s3-sns-sqs 1.6.1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2ebb0c135e09afcb7d44c4388817dec2050668e4255c05c62b869aa845f50648
-  data.tar.gz: 7b06d7101b9b6e5431b6014c71346b3f0edb7e12aff589f67f6f042786e87724
+  metadata.gz: 253c85cd1d1dfa22a59d282a0eeae4e5a71c5630473db65a768fe3e00131adc9
+  data.tar.gz: 5e98e0d9b47c7f9b47d6e11aefa2d5c14f59e4fe6c66a7bd0934a250bb8fbcfb
 SHA512:
-  metadata.gz: fad96d095a81b60159a6097cbefd1abd3e3924f9e6d2f77699cd73dc1be9db0cf1641249adaa778f9a38a968643970fae90f6f32e22679f564a360c47f5cb033
-  data.tar.gz: 404a3a25c01b391b85385f8aa427d80fc32aba34d0161b1a751f9bf7a0c19cf1bfcac132d3797a510132b100fba69b8eaa3aea7f6e802387f7ff7e52a4b9852b
+  metadata.gz: ee38fcc3de70af94b7de1570b054cdf435224c77b64bebd1db9b8eee3a4097b91f0320ce09d732fdefd8b1d8e0ca722c0a9b799e49dceda881e36bb7b26417e0
+  data.tar.gz: d6818a6bdead5aae583a09e2af6e1e869a7fee4902c6503def1150584afb16f415a825eb2a34b2e8a05130d8e6348bbda28f4885852e8eb92ec552e68382d15c

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+##2.0.0
+Breaking Changes:
+- s3_key_prefix was never functional and will be removed. Actually only used for metadata.folder backward compatibility.
+  config for s3 paths are regex (if not exact match)
+- s3_options_by_bucket substitutes all s3_* options
+  We will merge deprecated options into the new structure for one release
+Changes:
+- Refactor plugin structure to be more modular 
+- Rework threadding design
+- introduce s3_options_by_bucket to configure settings (e.g aws_options_hash or type)
 ##1.6.1
 - Fix typo in gzip error logging
 ##1.6.0

data/lib/logstash/inputs/codec_factory.rb

@@ -0,0 +1,37 @@
+# CodecFactory:
+# lazy-fetch codec plugins
+
+class CodecFactory
+  def initialize(logger, options)
+    @logger = logger
+    @default_codec = options[:default_codec]
+    @codec_by_folder = options[:codec_by_folder]
+    @codecs = {
+      'default' => @default_codec
+    }
+  end
+
+  def get_codec(record)
+    codec = find_codec(record)
+    if @codecs[codec].nil?
+      @codecs[codec] = get_codec_plugin(codec)
+    end
+    @logger.debug("Switching to codec #{codec}") if codec != 'default'
+    return @codecs[codec]
+  end
+
+  private
+
+  def find_codec(record)
+    bucket, key, folder = record[:bucket], record[:key], record[:folder]
+    unless @codec_by_folder[bucket].nil?
+      @logger.debug("trying to find codec for folder #{folder}", :codec =>  @codec_by_folder[bucket][folder])
+      return @codec_by_folder[bucket][folder] unless @codec_by_folder[bucket][folder].nil?
+    end
+    return 'default'
+  end
+
+  def get_codec_plugin(name, options = {})
+    LogStash::Plugin.lookup('codec', name).new(options)
+  end
+end

data/lib/logstash/inputs/s3/client_factory.rb

@@ -0,0 +1,61 @@
+# not needed - Mutex is part of core lib:
+#require 'thread'
+
+class S3ClientFactory
+
+  def initialize(logger, options, aws_options_hash)
+    @logger = logger
+    @aws_options_hash = aws_options_hash
+    @s3_default_options = Hash[options[:s3_default_options].map { |k, v| [k.to_sym, v] }]
+    @aws_options_hash.merge!(@s3_default_options) unless @s3_default_options.empty?
+    @sts_client = Aws::STS::Client.new(region: options[:aws_region])
+    @credentials_by_bucket = options[:s3_credentials_by_bucket]
+    @logger.debug("Credentials by Bucket", :credentials => @credentials_by_bucket)
+    @default_session_name = options[:s3_role_session_name]
+    @clients_by_bucket = {}
+    #@mutexes_by_bucket = {}
+    @creation_mutex = Mutex.new
+  end
+
+  def get_s3_client(bucket_name)
+    bucket_symbol = bucket_name.to_sym
+    @creation_mutex.synchronize do
+      if @clients_by_bucket[bucket_symbol].nil?
+        options = @aws_options_hash.clone
+        unless @credentials_by_bucket[bucket_name].nil?
+          options.merge!(credentials: get_s3_auth(@credentials_by_bucket[bucket_name]))
+        end
+        @clients_by_bucket[bucket_symbol] = Aws::S3::Client.new(options)
+        @logger.debug("Created a new S3 Client", :bucket_name => bucket_name, :client => @clients_by_bucket[bucket_symbol], :used_options => options)
+        #@mutexes_by_bucket[bucket_symbol] = Mutex.new
+      end
+    end
+    # to be thread-safe, one uses this method like this:
+    # s3_client_factory.get_s3_client(my_s3_bucket) do
+    #   ... do stuff ...
+    # end
+    # FIXME: this does not allow concurrent downloads from the same bucket!
+    #@mutexes_by_bucket[bucket_symbol].synchronize do
+    # So we are testing this without this mutex.
+    yield @clients_by_bucket[bucket_symbol]
+    #end
+  end
+
+  private
+
+  def get_s3_auth(credentials)
+    # reminder: these are auto-refreshing!
+    if credentials.key?('role')
+      @logger.debug("Assume Role", :role => credentials["role"])
+      return Aws::AssumeRoleCredentials.new(
+        client: @sts_client,
+        role_arn: credentials['role'],
+        role_session_name: @default_session_name
+      )
+    elsif credentials.key?('access_key_id') && credentials.key?('secret_access_key')
+      @logger.debug("Fetch credentials", :access_key => credentials['access_key_id'])
+      return Aws::Credentials.new(credentials)
+    end
+  end
+
+end # class

data/lib/logstash/inputs/s3/downloader.rb

@@ -0,0 +1,55 @@
+# encoding: utf-8
+require 'fileutils'
+require 'thread'
+
+class S3Downloader
+
+  def initialize(logger, stop_semaphore, options)
+    @logger = logger
+    @stopped = stop_semaphore
+    @factory = options[:s3_client_factory]
+    @delete_on_success = options[:delete_on_success]
+  end
+
+  def copy_s3object_to_disk(record)
+    # (from docs) WARNING:
+    # yielding data to a block disables retries of networking errors!
+    begin
+      @factory.get_s3_client(record[:bucket]) do |s3|
+        response = s3.get_object(
+          bucket: record[:bucket],
+          key: record[:key],
+          response_target: record[:local_file]
+        )
+      end
+    rescue Aws::S3::Errors::ServiceError => e
+      @logger.error("Unable to download file. Requeuing the message", :error => e, :record => record)
+      # prevent sqs message deletion
+      throw :skip_delete
+    end
+    throw :skip_delete if stop?
+    return true
+  end
+
+  def cleanup_local_object(record)
+    FileUtils.remove_entry_secure(record[:local_file], true) if ::File.exists?(record[:local_file])
+  rescue Exception => e
+    @logger.warn("Could not delete file", :file => record[:local_file], :error => e)
+  end
+
+  def cleanup_s3object(record)
+    return unless @delete_on_success
+    begin
+      @factory.get_s3_client(record[:bucket]) do |s3|
+        s3.delete_object(bucket: record[:bucket], key: record[:key])
+      end
+    rescue Exception => e
+      @logger.warn("Failed to delete s3 object", :record => record, :error => e)
+    end
+  end
+
+  def stop?
+    @stopped.value
+  end
+
+end # class

data/lib/logstash/inputs/s3snssqs.rb

@@ -1,15 +1,20 @@
 # encoding: utf-8
-#
 require "logstash/inputs/threadable"
 require "logstash/namespace"
 require "logstash/timestamp"
 require "logstash/plugin_mixins/aws_config"
+require "logstash/shutdown_watcher"
 require "logstash/errors"
 require 'logstash/inputs/s3sqs/patch'
 require "aws-sdk"
-require "stud/interval"
-require 'cgi'
-require 'logstash/inputs/mime/MagicgzipValidator'
+# "object-oriented interfaces on top of API clients"...
+# => Overhead. FIXME: needed?
+#require "aws-sdk-resources"
+require "fileutils"
+require "concurrent"
+# unused in code:
+#require "stud/interval"
+#require "digest/md5"
 
 require 'java'
 java_import java.io.InputStream
@@ -19,6 +24,14 @@ java_import java.io.BufferedReader
 java_import java.util.zip.GZIPInputStream
 java_import java.util.zip.ZipException
 
+# our helper classes
+# these may go into this file for brevity...
+require_relative 'sqs/poller'
+require_relative 's3/client_factory'
+require_relative 's3/downloader'
+require_relative 'codec_factory'
+require_relative 's3snssqs/log_processor'
+
 Aws.eager_autoload!
 
 # Get logs from AWS s3 buckets as issued by an object-created event via sqs.
@@ -89,472 +102,241 @@ Aws.eager_autoload!
 #
 class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   include LogStash::PluginMixins::AwsConfig::V2
-
-  BACKOFF_SLEEP_TIME = 1
-  BACKOFF_FACTOR = 2
-  MAX_TIME_BEFORE_GIVING_UP = 60
-  EVENT_SOURCE = 'aws:s3'
-  EVENT_TYPE = 'ObjectCreated'
+  include LogProcessor
 
   config_name "s3snssqs"
 
   default :codec, "plain"
 
+
+
+  # Future config might look somewhat like this:
+  #
+  # s3_options_by_bucket = [
+  #   {
+  #     "bucket_name": "my-beautiful-bucket",
+  #     "credentials": { "role": "aws:role:arn:for:bucket:access" },
+  #     "folders": [
+  #       {
+  #         "key": "my_folder",
+  #         "codec": "json"
+  #         "type": "my_lovely_index"
+  #       },
+  #       {
+  #         "key": "my_other_folder",
+  #         "codec": "json_stream"
+  #         "type": ""
+  #       }
+  #     ]
+  #   },
+  #   {
+  #     "bucket_name": "my-other-bucket"
+  #     "credentials": {
+  #        "access_key_id": "some-id",
+  #        "secret_access_key": "some-secret-key"
+  #     },
+  #     "folders": [
+  #       {
+  #         "key": ""
+  #       }
+  #     ]
+  #   }
+  # }
+
+  config :s3_key_prefix, :validate => :string, :default => '', :deprecated => true #, :obsolete => " Will be moved to s3_options_by_bucket/types"
+
+  config :s3_access_key_id, :validate => :string, :deprecated => true #, :obsolete => "Please migrate to :s3_options_by_bucket. We will remove this option in the next Version"
+  config :s3_secret_access_key, :validate => :string, :deprecated => true #, :obsolete => "Please migrate to :s3_options_by_bucket. We will remove this option in the next Version"
+  config :s3_role_arn, :validate => :string, :deprecated => true #, :obsolete => "Please migrate to :s3_options_by_bucket. We will remove this option in the next Version"
+
+  config :set_codec_by_folder, :validate => :hash, :default => {}, :deprecated => true #, :obsolete => "Please migrate to :s3_options_by_bucket. We will remove this option in the next Version"
+
+  # Default Options for the S3 clients
+  config :s3_default_options, :validate => :hash, :required => false, :default => {}
+  # We need a list of buckets, together with role arns and possible folder/codecs:
+  config :s3_options_by_bucket, :validate => :array, :required => false # TODO: true
+  # Session name to use when assuming an IAM role
+  config :s3_role_session_name, :validate => :string, :default => "logstash"
+
+  ### sqs
   # Name of the SQS Queue to pull messages from. Note that this is just the name of the queue, not the URL or ARN.
   config :queue, :validate => :string, :required => true
-  config :s3_key_prefix, :validate => :string, :default => ''
-  #Sometimes you need another key for s3. This is a first test...
-  config :s3_access_key_id, :validate => :string
-  config :s3_secret_access_key, :validate => :string
   config :queue_owner_aws_account_id, :validate => :string, :required => false
-  #If you have different file-types in you s3 bucket, you could define codec by folder
-  #set_codec_by_folder => {"My-ELB-logs" => "plain"}
-  config :set_codec_by_folder, :validate => :hash, :default => {}
-  config :delete_on_success, :validate => :boolean, :default => false
-  config :sqs_explicit_delete, :validate => :boolean, :default => false
   # Whether the event is processed though an SNS to SQS. (S3>SNS>SQS = true |S3>SQS=false)
   config :from_sns, :validate => :boolean, :default => true
-  # To run in multiple threads use this
-  config :consumer_threads, :validate => :number
-  config :temporary_directory, :validate => :string, :default => File.join(Dir.tmpdir, "logstash")
-  # The AWS IAM Role to assume, if any.
-  # This is used to generate temporary credentials typically for cross-account access.
-  # See https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html for more information.
-  config :s3_role_arn, :validate => :string
-  # Session name to use when assuming an IAM role
-  config :s3_role_session_name, :validate => :string, :default => "logstash"
+  config :sqs_skip_delete, :validate => :boolean, :default => false
+  config :delete_on_success, :validate => :boolean, :default => false
   config :visibility_timeout, :validate => :number, :default => 600
 
+  ### system
+  config :temporary_directory, :validate => :string, :default => File.join(Dir.tmpdir, "logstash")
+  # To run in multiple threads use this
+  config :consumer_threads, :validate => :number, :default => 1
 
-  attr_reader :poller
-  attr_reader :s3
-
-
-  def set_codec (folder)
-    begin
-      @logger.debug("Automatically switching from #{@codec.class.config_name} to #{set_codec_by_folder[folder]} codec", :plugin => self.class.config_name)
-      LogStash::Plugin.lookup("codec", "#{set_codec_by_folder[folder]}").new("charset" => @codec.charset)
-    rescue Exception => e
-      @logger.error("Failed to set_codec with error", :error => e)
-    end
-  end
 
   public
-  def register
-    require "fileutils"
-    require "digest/md5"
-    require "aws-sdk-resources"
 
-    @runner_threads = []
-    #make this hash keys lookups match like regex
-    hash_key_is_regex(set_codec_by_folder)
-    @logger.info("Registering SQS input", :queue => @queue)
-    setup_queue
+  # --- BEGIN plugin interface ----------------------------------------#
 
+  # initialisation
+  def register
+    # prepare system
     FileUtils.mkdir_p(@temporary_directory) unless Dir.exist?(@temporary_directory)
-  end
-
-  def setup_queue
-    aws_sqs_client = Aws::SQS::Client.new(aws_options_hash)
-    queue_url = aws_sqs_client.get_queue_url({ queue_name: @queue, queue_owner_aws_account_id: @queue_owner_aws_account_id})[:queue_url]
-    @poller = Aws::SQS::QueuePoller.new(queue_url, :client => aws_sqs_client)
-    get_s3client
-    @s3_resource = get_s3object
-  rescue Aws::SQS::Errors::ServiceError => e
-    @logger.error("Cannot establish connection to Amazon SQS", :error => e)
-    raise LogStash::ConfigurationError, "Verify the SQS queue name and your credentials"
-  end
-
-  def polling_options
-    {
-        # we will query 1 message at a time, so we can ensure correct error handling if we can't download a single file correctly
-        # (we will throw :skip_delete if download size isn't correct to process the event again later
-        # -> set a reasonable "Default Visibility Timeout" for your queue, so that there's enough time to process the log files)
-        :max_number_of_messages => 1,
-        # we will use the queue's setting, a good value is 10 seconds
-        # (to ensure fast logstash shutdown on the one hand and few api calls on the other hand)
-        :skip_delete => false,
-        :visibility_timeout => @visibility_timeout,
-        :wait_time_seconds => nil,
-    }
-  end
 
-  def handle_message(message, queue, instance_codec)
-    hash = JSON.parse message.body
-    @logger.debug("handle_message", :hash => hash, :message => message)
-    #If send via sns there is an additional JSON layer
-    if @from_sns then
-      hash = JSON.parse(hash['Message'])
-    end
-    # there may be test events sent from the s3 bucket which won't contain a Records array,
-    # we will skip those events and remove them from queue
-    if hash['Records'] then
-      # typically there will be only 1 record per event, but since it is an array we will
-      # treat it as if there could be more records
-      hash['Records'].each do |record|
-        @logger.debug("We found a record", :record => record)
-        # in case there are any events with Records that aren't s3 object-created events and can't therefore be
-        # processed by this plugin, we will skip them and remove them from queue
-        if record['eventSource'] == EVENT_SOURCE and record['eventName'].start_with?(EVENT_TYPE) then
-          @logger.debug("It is a valid record")
-          bucket = CGI.unescape(record['s3']['bucket']['name'])
-          key    = CGI.unescape(record['s3']['object']['key'])
-          size   = record['s3']['object']['size']
-          type_folder = get_object_folder(key)
-          # Set input codec by :set_codec_by_folder
-          instance_codec = set_codec(type_folder) unless set_codec_by_folder["#{type_folder}"].nil?
-          # try download and :skip_delete if it fails
-          #if record['s3']['object']['size'] < 10000000 then
-          process_log(bucket, key, type_folder, instance_codec, queue, message, size)
-          #else
-          #  @logger.info("Your file is too big")
-          #end
+    @credentials_by_bucket = hash_key_is_regex({})
+    # create the bucket=>folder=>codec lookup from config options
+    @codec_by_folder = hash_key_is_regex({})
+    @type_by_folder = hash_key_is_regex({})
+
+    # use deprecated settings only if new config is missing:
+    if @s3_options_by_bucket.nil?
+      # We don't know any bucket name, so we must rely on a "catch-all" regex
+      s3_options = {
+        'bucket_name' => '.*',
+        'folders' => @set_codec_by_folder.map { |key, codec|
+          { 'key' => key, 'codec' => codec }
+        }
+      }
+      if @s3_role_arn.nil?
+        # access key/secret key pair needed
+        unless @s3_access_key_id.nil? or @s3_secret_access_key.nil?
+          s3_options['credentials'] = {
+            'access_key_id' => @s3_access_key_id,
+            'secret_access_key' => @s3_secret_access_key
+          }
         end
+      else
+        s3_options['credentials'] = {
+          'role' => @s3_role_arn
+        }
       end
+      @s3_options_by_bucket = [s3_options]
     end
-  end
-
-  private
-  def process_log(bucket , key, folder, instance_codec, queue, message, size)
-    s3bucket = @s3_resource.bucket(bucket)
-    @logger.debug("Lets go reading file", :bucket => bucket, :key => key)
-    object = s3bucket.object(key)
-    filename = File.join(temporary_directory, File.basename(key))
-    if download_remote_file(object, filename)
-      if process_local_log( filename, key, folder, instance_codec, queue, bucket, message, size)
-        begin
-          FileUtils.remove_entry_secure(filename, true) if File.exists? filename
-          delete_file_from_bucket(object)
-        rescue Exception => e
-          @logger.debug("We had problems to delete your file", :file => filename, :error => e)
-        end
-      end
-    else
-      begin
-        FileUtils.remove_entry_secure(filename, true) if File.exists? filename
-      rescue Exception => e
-        @logger.debug("We had problems clean up your tmp dir", :file => filename, :error => e)
-      end
-    end
-  end
-
-  private
-  # Stream the remove file to the local disk
-  #
-  # @param [S3Object] Reference to the remove S3 objec to download
-  # @param [String] The Temporary filename to stream to.
-  # @return [Boolean] True if the file was completely downloaded
-  def download_remote_file(remote_object, local_filename)
-    completed = false
-    @logger.debug("S3 input: Download remote file", :remote_key => remote_object.key, :local_filename => local_filename)
-    File.open(local_filename, 'wb') do |s3file|
-      return completed if stop?
-      begin
-        remote_object.get(:response_target => s3file)
-      rescue Aws::S3::Errors::ServiceError => e
-        @logger.error("Unable to download file. We´ll requeue the message", :file => remote_object.inspect)
-        throw :skip_delete
-      end
-    end
-    completed = true
 
-    return completed
-  end
-
-  private
-
-  # Read the content of the local file
-  #
-  # @param [Queue] Where to push the event
-  # @param [String] Which file to read from
-  # @return [Boolean] True if the file was completely read, false otherwise.
-  def process_local_log(filename, key, folder, instance_codec, queue, bucket, message, size)
-    @logger.debug('Processing file', :filename => filename)
-    metadata = {}
-    start_time = Time.now
-    # Currently codecs operates on bytes instead of stream.
-    # So all IO stuff: decompression, reading need to be done in the actual
-    # input and send as bytes to the codecs.
-    read_file(filename) do |line|
-      if (Time.now - start_time) >= (@visibility_timeout.to_f / 100.0 * 90.to_f)
-        @logger.info("Increasing the visibility_timeout ... ", :timeout => @visibility_timeout, :filename => filename, :filesize => size, :start => start_time )
-        poller.change_message_visibility_timeout(message, @visibility_timeout)
-        start_time = Time.now
+    @s3_options_by_bucket.each do |options|
+      bucket = options['bucket_name']
+      if options.key?('credentials')
+        @credentials_by_bucket[bucket] = options['credentials']
       end
-      if stop?
-        @logger.warn("Logstash S3 input, stop reading in the middle of the file, we will read it again when logstash is started")
-        return false
-      end
-      line = line.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370")
-      #@logger.debug("read line", :line => line)
-      instance_codec.decode(line) do |event|
-        @logger.debug("decorate event")
-        # We are making an assumption concerning cloudfront
-        # log format, the user will use the plain or the line codec
-        # and the message key will represent the actual line content.
-        # If the event is only metadata the event will be drop.
-        # This was the behavior of the pre 1.5 plugin.
-        #
-        # The line need to go through the codecs to replace
-        # unknown bytes in the log stream before doing a regexp match or
-        # you will get a `Error: invalid byte sequence in UTF-8'
-        local_decorate_and_queue(event, queue, key, folder, metadata, bucket)
+      if options.key?('folders')
+        # make these hashes do key lookups using regex matching
+        folders = hash_key_is_regex({})
+        types = hash_key_is_regex({})
+        options['folders'].each do |entry|
+          @logger.debug("options for folder ", :folder => entry)
+          folders[entry['key']] = entry['codec'] if entry.key?('codec')
+          types[entry['key']] = entry['type'] if entry.key?('type')
+        end
+        @codec_by_folder[bucket] = folders unless folders.empty?
+        @type_by_folder[bucket] = types unless types.empty?
       end
     end
-    @logger.debug("end if file #{filename}")
-    #@logger.info("event pre flush", :event => event)
-    # #ensure any stateful codecs (such as multi-line ) are flushed to the queue
-    instance_codec.flush do |event|
-      local_decorate_and_queue(event, queue, key, folder, metadata, bucket)
-      @logger.debug("We´e to flush an incomplete event...", :event => event)
-    end
-
-    return true
-  end # def process_local_log
-
-  private
-  def local_decorate_and_queue(event, queue, key, folder, metadata, bucket)
-    @logger.debug('decorating event', :event => event.to_s)
-    if event_is_metadata?(event)
-      @logger.debug('Event is metadata, updating the current cloudfront metadata', :event => event)
-      update_metadata(metadata, event)
-    else
-
-      decorate(event)
-
-      event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
-      event.set("cloudfront_fields", metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
-
-      event.set("[@metadata][s3][object_key]", key)
-      event.set("[@metadata][s3][bucket_name]", bucket)
-      event.set("[@metadata][s3][object_folder]", folder)
-      @logger.debug('add metadata', :object_key => key, :bucket => bucket, :folder => folder)
-      queue << event
-    end
-  end
-
-
-  private
-  def get_object_folder(key)
-    if match=/#{s3_key_prefix}\/?(?<type_folder>.*?)\/.*/.match(key)
-      return match['type_folder']
-    else
-      return ""
-    end
-  end
-
-  private
-  def read_file(filename, &block)
-    if gzip?(filename)
-      read_gzip_file(filename, block)
-    else
-      read_plain_file(filename, block)
-    end
-  end
-
-  def read_plain_file(filename, block)
-    File.open(filename, 'rb') do |file|
-      file.each(&block)
-    end
-  end
-
-  private
-  def read_gzip_file(filename, block)
-    file_stream = FileInputStream.new(filename)
-    gzip_stream = GZIPInputStream.new(file_stream)
-    decoder = InputStreamReader.new(gzip_stream, "UTF-8")
-    buffered = BufferedReader.new(decoder)
-
-    while (line = buffered.readLine())
-      block.call(line)
-    end
-  rescue ZipException => e
-    @logger.error("Gzip codec: We cannot uncompress the gzip file", :filename => filename, :error => e)
-  ensure
-    buffered.close unless buffered.nil?
-    decoder.close unless decoder.nil?
-    gzip_stream.close unless gzip_stream.nil?
-    file_stream.close unless file_stream.nil?
-  end
-
-  private
-  def gzip?(filename)
-    return true if filename.end_with?('.gz','.gzip')
-    MagicGzipValidator.new(File.new(filename, 'r')).valid?
-  rescue Exception => e
-    @logger.debug("Problem while gzip detection", :error => e)
-  end
-
-  private
-  def delete_file_from_bucket(object)
-    if @delete_on_success
-      object.delete()
-    end
-  end
-
-
-  private
-  def get_s3client
-    if s3_access_key_id and s3_secret_access_key
-      @logger.debug("Using S3 Credentials from config", :ID => aws_options_hash.merge(:access_key_id => s3_access_key_id, :secret_access_key => s3_secret_access_key) )
-      @s3_client = Aws::S3::Client.new(aws_options_hash.merge(:access_key_id => s3_access_key_id, :secret_access_key => s3_secret_access_key))
-    elsif @s3_role_arn
-      @s3_client = Aws::S3::Client.new(aws_options_hash.merge!({ :credentials => s3_assume_role }))
-      @logger.debug("Using S3 Credentials from role", :s3client => @s3_client.inspect, :options => aws_options_hash.merge!({ :credentials => s3_assume_role }))
-    else
-      @s3_client = Aws::S3::Client.new(aws_options_hash)
-    end
-  end
-
-  private
-  def get_s3object
-    s3 = Aws::S3::Resource.new(client: @s3_client)
-  end
-
-  private
-  def s3_assume_role()
-    Aws::AssumeRoleCredentials.new(
-        client: Aws::STS::Client.new(region: @region),
-        role_arn: @s3_role_arn,
-        role_session_name: @s3_role_session_name
-    )
-  end
-
-  private
-  def event_is_metadata?(event)
-    return false unless event.get("message").class == String
-    line = event.get("message")
-    version_metadata?(line) || fields_metadata?(line)
-  end
-
-  private
-  def version_metadata?(line)
-    line.start_with?('#Version: ')
-  end
-
-  private
-  def fields_metadata?(line)
-    line.start_with?('#Fields: ')
-  end
-
-  private
-  def update_metadata(metadata, event)
-    line = event.get('message').strip
-
-    if version_metadata?(line)
-      metadata[:cloudfront_version] = line.split(/#Version: (.+)/).last
-    end
 
-    if fields_metadata?(line)
-      metadata[:cloudfront_fields] = line.split(/#Fields: (.+)/).last
-    end
+    @received_stop = Concurrent::AtomicBoolean.new(false)
+
+    # instantiate helpers
+    @sqs_poller = SqsPoller.new(@logger, @received_stop, @queue, {
+      queue_owner_aws_account_id: @queue_owner_aws_account_id,
+      from_sns: @from_sns,
+      sqs_explicit_delete: @sqs_explicit_delete,
+      visibility_timeout: @visibility_timeout
+    }, aws_options_hash)
+    @s3_client_factory = S3ClientFactory.new(@logger, {
+      aws_region: @region,
+      s3_default_options: @s3_default_options,
+      s3_credentials_by_bucket: @credentials_by_bucket,
+      s3_role_session_name: @s3_role_session_name
+    }, aws_options_hash)
+    @s3_downloader = S3Downloader.new(@logger, @received_stop, {
+      s3_client_factory: @s3_client_factory,
+      delete_on_success: @delete_on_success
+    })
+    @codec_factory = CodecFactory.new(@logger, {
+      default_codec: @codec,
+      codec_by_folder: @codec_by_folder
+    })
+    #@log_processor = LogProcessor.new(self)
+
+    # administrative stuff
+    @worker_threads = []
   end
 
-  public
-  def run(queue)
-    if @consumer_threads
-      # ensure we can stop logstash correctly
-      @runner_threads = consumer_threads.times.map { |consumer| thread_runner(queue) }
-      @runner_threads.each { |t| t.join }
-    else
-      #Fallback to simple single thread worker
-      # ensure we can stop logstash correctly
-      poller.before_request do |stats|
-        if stop? then
-          @logger.warn("issuing :stop_polling on stop?", :queue => @queue)
-          # this can take up to "Receive Message Wait Time" (of the sqs queue) seconds to be recognized
-          throw :stop_polling
-        end
-      end
-      # poll a message and process it
-      run_with_backoff do
-        poller.poll(polling_options) do |message|
-          begin
-            handle_message(message, queue, @codec.clone)
-            poller.delete_message(message)
-          rescue Exception => e
-            @logger.info("Error in poller block ... ", :error => e)
-          end
-        end
-      end
+  # startup
+  def run(logstash_event_queue)
+    #LogStash::ShutdownWatcher.abort_threshold(30)
+    # start them
+    @worker_threads = @consumer_threads.times.map do |_|
+      run_worker_thread(logstash_event_queue)
     end
+    # and wait (possibly infinitely) for them to shut down
+    @worker_threads.each { |t| t.join }
   end
 
-  public
+  # shutdown
   def stop
-    if @consumer_threads
-      @runner_threads.each do |c|
-        begin
-          @logger.info("Stopping thread ... ", :thread => c.inspect)
-          c.wakeup
-        rescue
-          @logger.error("Cannot stop thread ... try to kill him", :thread => c.inspect)
-          c.kill
-        end
+    @received_stop.make_true
+    @worker_threads.each do |worker|
+      begin
+        @logger.info("Stopping thread ... ", :thread => worker.inspect)
+        worker.wakeup
+      rescue
+        @logger.error("Cannot stop thread ... try to kill him", :thread => worker.inspect)
+        worker.kill
       end
-    else
-      @logger.warn("Stopping all threads?", :queue => @queue)
     end
   end
 
+  # --- END plugin interface ------------------------------------------#
+
   private
-  def thread_runner(queue)
+
+  def run_worker_thread(queue)
     Thread.new do
-      @logger.info("Starting new thread")
-      begin
-        poller.before_request do |stats|
-          if stop? then
-            @logger.warn("issuing :stop_polling on stop?", :queue => @queue)
-            # this can take up to "Receive Message Wait Time" (of the sqs queue) seconds to be recognized
-            throw :stop_polling
-          end
-        end
-        # poll a message and process it
-        run_with_backoff do
-          poller.poll(polling_options) do |message|
-            begin
-              handle_message(message, queue, @codec.clone)
-              poller.delete_message(message) if @sqs_explicit_delete
-            rescue Exception => e
-              @logger.info("Error in poller block ... ", :error => e)
-            end
+      @logger.info("Starting new worker thread")
+      @sqs_poller.run do |record|
+        throw :skip_delete if stop?
+        @logger.debug("Outside Poller: got a record", :record => record)
+        # record is a valid object with the keys ":bucket", ":key", ":size"
+        record[:local_file] = File.join(@temporary_directory, File.basename(record[:key]))
+        if @s3_downloader.copy_s3object_to_disk(record)
+          completed = catch(:skip_delete) do
+            process(record, queue)
           end
+          @s3_downloader.cleanup_local_object(record)
+          # re-throw if necessary:
+          throw :skip_delete unless completed
+          @s3_downloader.cleanup_s3object(record)
         end
       end
     end
   end
 
-  private
-  # Runs an AWS request inside a Ruby block with an exponential backoff in case
-  # we experience a ServiceError.
-  #
-  # @param [Integer] max_time maximum amount of time to sleep before giving up.
-  # @param [Integer] sleep_time the initial amount of time to sleep before retrying.
-  # @param [Block] block Ruby code block to execute.
-  def run_with_backoff(max_time = MAX_TIME_BEFORE_GIVING_UP, sleep_time = BACKOFF_SLEEP_TIME, &block)
-    next_sleep = sleep_time
-    begin
-      block.call
-      next_sleep = sleep_time
-    rescue Aws::SQS::Errors::ServiceError => e
-      @logger.warn("Aws::SQS::Errors::ServiceError ... retrying SQS request with exponential backoff", :queue => @queue, :sleep_time => sleep_time, :error => e)
-      sleep(next_sleep)
-      next_sleep =  next_sleep > max_time ? sleep_time : sleep_time * BACKOFF_FACTOR
-      retry
+  # Will be remove in further releases...
+  def get_object_folder(key)
+    if match=/#{s3_key_prefix}\/?(?<type_folder>.*?)\/.*/.match(key)
+      return match['type_folder']
+    else
+      return ""
     end
   end
 
-  private
   def hash_key_is_regex(myhash)
     myhash.default_proc = lambda do |hash, lookup|
       result=nil
       hash.each_pair do |key, value|
         if %r[#{key}] =~ lookup
-          result=value
+          result = value
           break
         end
       end
       result
     end
+    # return input hash (convenience)
+    return myhash
+  end
+
+  def stop?
+    @received_stop.value
   end
+
 end # class