logstash-input-s3-test 4.0.0

data/docs/index.asciidoc

@@ -0,0 +1,317 @@
+:plugin: s3
+:type: input
+:default_codec: plain
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}s-{plugin}"]
+
+=== S3 input plugin
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+Stream events from files from a S3 bucket.
+
+IMPORTANT: The S3 input plugin only supports AWS S3. 
+Other S3 compatible storage solutions are not supported.
+
+Each line from each file generates an event.
+Files ending in `.gz` are handled as gzip'ed files.
+
+Files that are archived to AWS Glacier will be skipped.
+
+[id="plugins-{type}s-{plugin}-options"]
+==== S3 Input Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-access_key_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-additional_settings>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-aws_credentials_file>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-backup_add_prefix>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-backup_to_bucket>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-backup_to_dir>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-bucket>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-delete>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-endpoint>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-exclude_pattern>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-gzip_pattern>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-include_object_properties>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-interval>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-prefix>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-proxy_uri>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-region>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-role_arn>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-role_session_name>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-secret_access_key>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-session_token>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-sincedb_path>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-temporary_directory>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-watch_for_new_files>> |<<boolean,boolean>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+input plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-access_key_id"]
+===== `access_key_id` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order:
+
+1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config
+2. External credentials file specified by `aws_credentials_file`
+3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY`
+5. IAM Instance Profile (available when running inside EC2)
+
+
+[id="plugins-{type}s-{plugin}-additional_settings"]
+===== `additional_settings`
+
+  * Value type is <<hash,hash>>
+  * Default value is `{}`
+
+Key-value pairs of settings and corresponding values used to parametrize
+the connection to s3. See full list in https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html[the AWS SDK documentation]. Example:
+
+[source,ruby]
+    input {
+      s3 {
+        "access_key_id" => "1234"
+        "secret_access_key" => "secret"
+        "bucket" => "logstash-test"
+        "additional_settings" => {
+          "force_path_style" => true
+          "follow_redirects" => false
+        }
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-aws_credentials_file"]
+===== `aws_credentials_file` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Path to YAML file containing a hash of AWS credentials.
+This file will only be loaded if `access_key_id` and
+`secret_access_key` aren't set. The contents of the
+file should look like this:
+
+[source,ruby]
+----------------------------------
+    :access_key_id: "12345"
+    :secret_access_key: "54321"
+----------------------------------
+
+
+[id="plugins-{type}s-{plugin}-backup_add_prefix"]
+===== `backup_add_prefix` 
+
+  * Value type is <<string,string>>
+  * Default value is `nil`
+
+Append a prefix to the key (full path including file name in s3) after processing.
+If backing up to another (or the same) bucket, this effectively lets you
+choose a new 'folder' to place the files in
+
+[id="plugins-{type}s-{plugin}-backup_to_bucket"]
+===== `backup_to_bucket` 
+
+  * Value type is <<string,string>>
+  * Default value is `nil`
+
+Name of a S3 bucket to backup processed files to.
+
+[id="plugins-{type}s-{plugin}-backup_to_dir"]
+===== `backup_to_dir` 
+
+  * Value type is <<string,string>>
+  * Default value is `nil`
+
+Path of a local directory to backup processed files to.
+
+[id="plugins-{type}s-{plugin}-bucket"]
+===== `bucket` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The name of the S3 bucket.
+
+[id="plugins-{type}s-{plugin}-delete"]
+===== `delete` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Whether to delete processed files from the original bucket.
+
+[id="plugins-{type}s-{plugin}-endpoint"]
+===== `endpoint`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The endpoint to connect to. By default it is constructed using the value of `region`.
+This is useful when connecting to S3 compatible services, but beware that these aren't
+guaranteed to work correctly with the AWS SDK.
+
+[id="plugins-{type}s-{plugin}-exclude_pattern"]
+===== `exclude_pattern` 
+
+  * Value type is <<string,string>>
+  * Default value is `nil`
+
+Ruby style regexp of keys to exclude from the bucket.
+
+Note that files matching the pattern are skipped _after_ they have been listed. 
+Consider using <<plugins-{type}s-{plugin}-prefix>> instead where possible.
+
+Example:
+
+[source,ruby]
+-----
+"exclude_pattern" => "\/2020\/04\/"
+-----
+
+This pattern excludes all logs containing "/2020/04/" in the path.
+
+
+[id="plugins-{type}s-{plugin}-gzip_pattern"]
+===== `gzip_pattern`
+
+  * Value type is <<string,string>>
+  * Default value is `"\.gz(ip)?$"`
+
+Regular expression used to determine whether an input file is in gzip format.
+
+[id="plugins-{type}s-{plugin}-include_object_properties"]
+===== `include_object_properties`
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Whether or not to include the S3 object's properties (last_modified, content_type, metadata) into each Event at
+`[@metadata][s3]`. Regardless of this setting, `[@metadata][s3][key]` will always be present.
+
+[id="plugins-{type}s-{plugin}-interval"]
+===== `interval` 
+
+  * Value type is <<number,number>>
+  * Default value is `60`
+
+Interval to wait between to check the file list again after a run is finished.
+Value is in seconds.
+
+[id="plugins-{type}s-{plugin}-prefix"]
+===== `prefix` 
+
+  * Value type is <<string,string>>
+  * Default value is `nil`
+
+If specified, the prefix of filenames in the bucket must match (not a regexp)
+
+[id="plugins-{type}s-{plugin}-proxy_uri"]
+===== `proxy_uri` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+URI to proxy server if required
+
+[id="plugins-{type}s-{plugin}-region"]
+===== `region` 
+
+  * Value type is <<string,string>>
+  * Default value is `"us-east-1"`
+
+The AWS Region
+
+[id="plugins-{type}s-{plugin}-role_arn"]
+===== `role_arn`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS IAM Role to assume, if any.
+This is used to generate temporary credentials, typically for cross-account access.
+See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information.
+
+[id="plugins-{type}s-{plugin}-role_session_name"]
+===== `role_session_name`
+
+  * Value type is <<string,string>>
+  * Default value is `"logstash"`
+
+Session name to use when assuming an IAM role.
+
+[id="plugins-{type}s-{plugin}-secret_access_key"]
+===== `secret_access_key` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS Secret Access Key
+
+[id="plugins-{type}s-{plugin}-session_token"]
+===== `session_token` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS Session token for temporary credential
+
+[id="plugins-{type}s-{plugin}-sincedb_path"]
+===== `sincedb_path` 
+
+  * Value type is <<string,string>>
+  * Default value is `nil`
+
+Where to write the since database (keeps track of the date
+the last handled file was added to S3). The default will write
+sincedb files to in the directory '{path.data}/plugins/inputs/s3/'
+
+If specified, this setting must be a filename path and not just a directory.
+
+[id="plugins-{type}s-{plugin}-temporary_directory"]
+===== `temporary_directory` 
+
+  * Value type is <<string,string>>
+  * Default value is `"/tmp/logstash"`
+
+Set the directory where logstash will store the tmp files before processing them.
+
+[id="plugins-{type}s-{plugin}-watch_for_new_files"]
+===== `watch_for_new_files`
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+Whether or not to watch for new files.
+Disabling this option causes the input to close itself after processing the files from a single listing.
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]
+
+:default_codec!:

data/lib/logstash/inputs/s3.rb

@@ -0,0 +1,545 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "time"
+require "date"
+require "tmpdir"
+require "stud/interval"
+require "stud/temporary"
+require "aws-sdk-s3"
+require "logstash/inputs/s3/patch"
+
+require 'java'
+
+Aws.eager_autoload!
+# Stream events from files from a S3 bucket.
+#
+# Each line from each file generates an event.
+# Files ending in `.gz` are handled as gzip'ed files.
+class LogStash::Inputs::S3 < LogStash::Inputs::Base
+
+  java_import java.io.InputStream
+  java_import java.io.InputStreamReader
+  java_import java.io.FileInputStream
+  java_import java.io.BufferedReader
+  java_import java.util.zip.GZIPInputStream
+  java_import java.util.zip.ZipException
+
+  CredentialConfig = Struct.new(
+    :access_key_id,
+    :secret_access_key,
+    :session_token,
+    :profile,
+    :instance_profile_credentials_retries,
+    :instance_profile_credentials_timeout,
+    :region)
+
+  config_name "s3"
+
+  default :codec, "plain"
+
+  # The name of the S3 bucket.
+  config :bucket, :validate => :string, :required => true
+
+  # If specified, the prefix of filenames in the bucket must match (not a regexp)
+  config :prefix, :validate => :string, :default => nil
+
+  config :additional_settings, :validate => :hash, :default => {}
+
+  # The path to use for writing state. The state stored by this plugin is
+  # a memory of files already processed by this plugin.
+  #
+  # If not specified, the default is in `{path.data}/plugins/inputs/s3/...`
+  #
+  # Should be a path with filename not just a directory.
+  config :sincedb_path, :validate => :string, :default => nil
+
+  # Name of a S3 bucket to backup processed files to.
+  config :backup_to_bucket, :validate => :string, :default => nil
+
+  # Append a prefix to the key (full path including file name in s3) after processing.
+  # If backing up to another (or the same) bucket, this effectively lets you
+  # choose a new 'folder' to place the files in
+  config :backup_add_prefix, :validate => :string, :default => nil
+
+  # Path of a local directory to backup processed files to.
+  config :backup_to_dir, :validate => :string, :default => nil
+
+  # Whether to delete processed files from the original bucket.
+  config :delete, :validate => :boolean, :default => false
+
+  # Interval to wait between to check the file list again after a run is finished.
+  # Value is in seconds.
+  config :interval, :validate => :number, :default => 60
+
+  # Whether to watch for new files with the interval.
+  # If false, overrides any interval and only lists the s3 bucket once.
+  config :watch_for_new_files, :validate => :boolean, :default => true
+
+  # Ruby style regexp of keys to exclude from the bucket
+  config :exclude_pattern, :validate => :string, :default => nil
+
+  # Set the directory where logstash will store the tmp files before processing them.
+  # default to the current OS temporary directory in linux /tmp/logstash
+  config :temporary_directory, :validate => :string, :default => File.join(Dir.tmpdir, "logstash")
+
+  # Whether or not to include the S3 object's properties (last_modified, content_type, metadata)
+  # into each Event at [@metadata][s3]. Regardless of this setting, [@metdata][s3][key] will always
+  # be present.
+  config :include_object_properties, :validate => :boolean, :default => false
+
+  # Regular expression used to determine whether an input file is in gzip format.
+  # default to an expression that matches *.gz and *.gzip file extensions
+  config :gzip_pattern, :validate => :string, :default => "\.gz(ip)?$"
+
+  config :region, :validate => :string, :default => "us-east-1"
+
+  # This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order:
+  #
+  # 1. Static configuration, using `access_key_id` and `secret_access_key` params or `role_arn` in the logstash plugin config
+  # 2. External credentials file specified by `aws_credentials_file`
+  # 3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+  # 4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY`
+  # 5. IAM Instance Profile (available when running inside EC2)
+  config :access_key_id, :validate => :string
+
+  # The AWS Secret Access Key
+  config :secret_access_key, :validate => :string
+
+  # Profile
+  config :profile, :validate => :string, :default => "default"
+
+  # The AWS Session token for temporary credential
+  config :session_token, :validate => :password
+
+  # URI to proxy server if required
+  config :proxy_uri, :validate => :string
+
+  # Custom endpoint to connect to s3
+  config :endpoint, :validate => :string
+
+  # The AWS IAM Role to assume, if any.
+  # This is used to generate temporary credentials typically for cross-account access.
+  # See https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html for more information.
+  config :role_arn, :validate => :string
+
+  # Session name to use when assuming an IAM role
+  config :role_session_name, :validate => :string, :default => "logstash"
+
+  # Path to YAML file containing a hash of AWS credentials.
+  # This file will only be loaded if `access_key_id` and
+  # `secret_access_key` aren't set. The contents of the
+  # file should look like this:
+  #
+  # [source,ruby]
+  # ----------------------------------
+  #     :access_key_id: "12345"
+  #     :secret_access_key: "54321"
+  # ----------------------------------
+  #
+  config :aws_credentials_file, :validate => :string
+
+  def register
+    require "fileutils"
+    require "digest/md5"
+
+    @logger.info("Registering", :bucket => @bucket, :region => @region)
+
+    s3 = get_s3object
+
+    @s3bucket = s3.bucket(@bucket)
+
+    unless @backup_to_bucket.nil?
+      @backup_bucket = s3.bucket(@backup_to_bucket)
+      begin
+        s3.client.head_bucket({ :bucket => @backup_to_bucket})
+      rescue Aws::S3::Errors::NoSuchBucket
+        s3.create_bucket({ :bucket => @backup_to_bucket})
+      end
+    end
+
+    unless @backup_to_dir.nil?
+      Dir.mkdir(@backup_to_dir, 0700) unless File.exists?(@backup_to_dir)
+    end
+
+    FileUtils.mkdir_p(@temporary_directory) unless Dir.exist?(@temporary_directory)
+
+    if !@watch_for_new_files && original_params.include?('interval')
+      logger.warn("`watch_for_new_files` has been disabled; `interval` directive will be ignored.")
+    end
+  end
+
+  def run(queue)
+    @current_thread = Thread.current
+    Stud.interval(@interval) do
+      process_files(queue)
+      stop unless @watch_for_new_files
+    end
+  end # def run
+
+  def list_new_files
+    objects = {}
+    found = false
+    begin
+      @s3bucket.objects(:prefix => @prefix).each do |log|
+        found = true
+        @logger.debug('Found key', :key => log.key)
+        if ignore_filename?(log.key)
+          @logger.debug('Ignoring', :key => log.key)
+        elsif log.content_length <= 0
+          @logger.debug('Object Zero Length', :key => log.key)
+        elsif !sincedb.newer?(log.last_modified)
+          @logger.debug('Object Not Modified', :key => log.key)
+        elsif (log.storage_class == 'GLACIER' || log.storage_class == 'DEEP_ARCHIVE') && !file_restored?(log.object)
+          @logger.debug('Object Archived to Glacier', :key => log.key)
+        else
+          objects[log.key] = log.last_modified
+          @logger.debug("Added to objects[]", :key => log.key, :length => objects.length)
+        end
+      end
+      @logger.info('No files found in bucket', :prefix => prefix) unless found
+    rescue Aws::Errors::ServiceError => e
+      @logger.error("Unable to list objects in bucket", :exception => e.class, :message => e.message, :backtrace => e.backtrace, :prefix => prefix)
+    end
+    objects.keys.sort {|a,b| objects[a] <=> objects[b]}
+  end # def fetch_new_files
+
+  def backup_to_bucket(object)
+    unless @backup_to_bucket.nil?
+      backup_key = "#{@backup_add_prefix}#{object.key}"
+      @backup_bucket.object(backup_key).copy_from(:copy_source => "#{object.bucket_name}/#{object.key}")
+      if @delete
+        object.delete()
+      end
+    end
+  end
+
+  def backup_to_dir(filename)
+    unless @backup_to_dir.nil?
+      FileUtils.cp(filename, @backup_to_dir)
+    end
+  end
+
+  def process_files(queue)
+    objects = list_new_files
+
+    objects.each do |key|
+      if stop?
+        break
+      else
+        process_log(queue, key)
+      end
+    end
+  end # def process_files
+
+  def stop
+    # @current_thread is initialized in the `#run` method,
+    # this variable is needed because the `#stop` is a called in another thread
+    # than the `#run` method and requiring us to call stop! with a explicit thread.
+    Stud.stop!(@current_thread)
+  end
+
+  private
+
+  # Read the content of the local file
+  #
+  # @param [Queue] Where to push the event
+  # @param [String] Which file to read from
+  # @param [S3Object] Source s3 object
+  # @return [Boolean] True if the file was completely read, false otherwise.
+  def process_local_log(queue, filename, object)
+    @logger.debug('Processing file', :filename => filename)
+    metadata = {}
+    # Currently codecs operates on bytes instead of stream.
+    # So all IO stuff: decompression, reading need to be done in the actual
+    # input and send as bytes to the codecs.
+    read_file(filename) do |line|
+      if stop?
+        @logger.warn("Logstash S3 input, stop reading in the middle of the file, we will read it again when logstash is started")
+        return false
+      end
+
+      @codec.decode(line) do |event|
+        # We are making an assumption concerning cloudfront
+        # log format, the user will use the plain or the line codec
+        # and the message key will represent the actual line content.
+        # If the event is only metadata the event will be drop.
+        # This was the behavior of the pre 1.5 plugin.
+        #
+        # The line need to go through the codecs to replace
+        # unknown bytes in the log stream before doing a regexp match or
+        # you will get a `Error: invalid byte sequence in UTF-8'
+        if event_is_metadata?(event)
+          @logger.debug('Event is metadata, updating the current cloudfront metadata', :event => event)
+          update_metadata(metadata, event)
+        else
+          decorate(event)
+
+          event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
+          event.set("cloudfront_fields", metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
+
+          if @include_object_properties
+            event.set("[@metadata][s3]", object.data.to_h)
+          else
+            event.set("[@metadata][s3]", {})
+          end
+
+          event.set("[@metadata][s3][key]", object.key)
+
+          queue << event
+        end
+      end
+    end
+    # #ensure any stateful codecs (such as multi-line ) are flushed to the queue
+    @codec.flush do |event|
+      queue << event
+    end
+
+    return true
+  end # def process_local_log
+
+  def event_is_metadata?(event)
+    return false unless event.get("message").class == String
+    line = event.get("message")
+    version_metadata?(line) || fields_metadata?(line)
+  end
+
+  def version_metadata?(line)
+    line.start_with?('#Version: ')
+  end
+
+  def fields_metadata?(line)
+    line.start_with?('#Fields: ')
+  end
+
+  def update_metadata(metadata, event)
+    line = event.get('message').strip
+
+    if version_metadata?(line)
+      metadata[:cloudfront_version] = line.split(/#Version: (.+)/).last
+    end
+
+    if fields_metadata?(line)
+      metadata[:cloudfront_fields] = line.split(/#Fields: (.+)/).last
+    end
+  end
+
+  def read_file(filename, &block)
+    if gzip?(filename)
+      read_gzip_file(filename, block)
+    else
+      read_plain_file(filename, block)
+    end
+  rescue => e
+    # skip any broken file
+    @logger.error("Failed to read file, processing skipped", :exception => e.class, :message => e.message, :filename => filename)
+  end
+
+  def read_plain_file(filename, block)
+    File.open(filename, 'rb') do |file|
+      file.each(&block)
+    end
+  end
+
+  def read_gzip_file(filename, block)
+    file_stream = FileInputStream.new(filename)
+    gzip_stream = GZIPInputStream.new(file_stream)
+    decoder = InputStreamReader.new(gzip_stream, "UTF-8")
+    buffered = BufferedReader.new(decoder)
+
+    while (line = buffered.readLine())
+      block.call(line)
+    end
+  ensure
+    buffered.close unless buffered.nil?
+    decoder.close unless decoder.nil?
+    gzip_stream.close unless gzip_stream.nil?
+    file_stream.close unless file_stream.nil?
+  end
+
+  def gzip?(filename)
+    Regexp.new(@gzip_pattern).match(filename)
+  end
+
+  def sincedb
+    @sincedb ||= if @sincedb_path.nil?
+                    @logger.info("Using default generated file for the sincedb", :filename => sincedb_file)
+                    SinceDB::File.new(sincedb_file)
+                  else
+                    @logger.info("Using the provided sincedb_path", :sincedb_path => @sincedb_path)
+                    SinceDB::File.new(@sincedb_path)
+                  end
+  end
+
+  def sincedb_file
+    digest = Digest::MD5.hexdigest("#{@bucket}+#{@prefix}")
+    dir = File.join(LogStash::SETTINGS.get_value("path.data"), "plugins", "inputs", "s3")
+    FileUtils::mkdir_p(dir)
+    path = File.join(dir, "sincedb_#{digest}")
+
+    # Migrate old default sincedb path to new one.
+    if ENV["HOME"]
+      # This is the old file path including the old digest mechanism.
+      # It remains as a way to automatically upgrade users with the old default ($HOME)
+      # to the new default (path.data)
+      old = File.join(ENV["HOME"], ".sincedb_" + Digest::MD5.hexdigest("#{@bucket}+#{@prefix}"))
+      if File.exist?(old)
+        logger.info("Migrating old sincedb in $HOME to {path.data}")
+        FileUtils.mv(old, path)
+      end
+    end
+
+    path
+  end
+
+  def symbolized_settings
+    @symbolized_settings ||= symbolize(@additional_settings)
+  end
+
+  def symbolize(hash)
+    return hash unless hash.is_a?(Hash)
+    symbolized = {}
+    hash.each { |key, value| symbolized[key.to_sym] = symbolize(value) }
+    symbolized
+  end
+
+  def ignore_filename?(filename)
+    if @prefix == filename
+      return true
+    elsif filename.end_with?("/")
+      return true
+    elsif (@backup_add_prefix && @backup_to_bucket == @bucket && filename =~ /^#{backup_add_prefix}/)
+      return true
+    elsif @exclude_pattern.nil?
+      return false
+    elsif filename =~ Regexp.new(@exclude_pattern)
+      return true
+    else
+      return false
+    end
+  end
+
+  def process_log(queue, key)
+    @logger.debug("Processing", :bucket => @bucket, :key => key)
+    object = @s3bucket.object(key)
+
+    filename = File.join(temporary_directory, File.basename(key))
+    if download_remote_file(object, filename)
+      if process_local_log(queue, filename, object)
+        lastmod = object.last_modified
+        backup_to_bucket(object)
+        backup_to_dir(filename)
+        delete_file_from_bucket(object)
+        FileUtils.remove_entry_secure(filename, true)
+        sincedb.write(lastmod)
+      end
+    else
+      FileUtils.remove_entry_secure(filename, true)
+    end
+  end
+
+  # Stream the remove file to the local disk
+  #
+  # @param [S3Object] Reference to the remove S3 objec to download
+  # @param [String] The Temporary filename to stream to.
+  # @return [Boolean] True if the file was completely downloaded
+  def download_remote_file(remote_object, local_filename)
+    completed = false
+    @logger.debug("Downloading remote file", :remote_key => remote_object.key, :local_filename => local_filename)
+    File.open(local_filename, 'wb') do |s3file|
+      return completed if stop?
+      begin
+        remote_object.get(:response_target => s3file)
+        completed = true
+      rescue Aws::Errors::ServiceError => e
+        @logger.warn("Unable to download remote file", :exception => e.class, :message => e.message, :remote_key => remote_object.key)
+      end
+    end
+    completed
+  end
+
+  def delete_file_from_bucket(object)
+    if @delete and @backup_to_bucket.nil?
+      object.delete()
+    end
+  end
+
+  def aws_options_hash
+    opts = {}
+
+    if @access_key_id.is_a?(NilClass) ^ @secret_access_key.is_a?(NilClass)
+      @logger.warn("Likely config error: Only one of access_key_id or secret_access_key was provided but not both.")
+    end
+
+    credential_config = CredentialConfig.new(@access_key_id, @secret_access_key, @session_token, @profile, 0, 1, @region)
+    @credentials = Aws::CredentialProviderChain.new(credential_config).resolve
+
+    opts[:credentials] = @credentials
+
+    opts[:http_proxy] = @proxy_uri if @proxy_uri
+
+    if self.respond_to?(:aws_service_endpoint)
+      # used by CloudWatch to basically do the same as bellow (returns { region: region })
+      opts.merge!(self.aws_service_endpoint(@region))
+    else
+      # NOTE: setting :region works with the aws sdk (resolves correct endpoint)
+      opts[:region] = @region
+    end
+
+    if !@endpoint.is_a?(NilClass)
+      opts[:endpoint] = @endpoint
+    end
+
+    return opts
+  end
+
+  def get_s3object
+    options = symbolized_settings.merge(aws_options_hash || {})
+    s3 = Aws::S3::Resource.new(options)
+  end
+
+  def file_restored?(object)
+    begin
+      restore = object.data.restore
+      if restore && restore.match(/ongoing-request\s?=\s?["']false["']/)
+        if restore = restore.match(/expiry-date\s?=\s?["'](.*?)["']/)
+          expiry_date = DateTime.parse(restore[1])
+          return true if DateTime.now < expiry_date # restored
+        else
+          @logger.debug("No expiry-date header for restore request: #{object.data.restore}")
+          return nil # no expiry-date found for ongoing request
+        end
+      end
+    rescue => e
+      @logger.debug("Could not determine Glacier restore status", :exception => e.class, :message => e.message)
+    end
+    return false
+  end
+
+  module SinceDB
+    class File
+      def initialize(file)
+        @sincedb_path = file
+      end
+
+      def newer?(date)
+        date > read
+      end
+
+      def read
+        if ::File.exists?(@sincedb_path)
+          content = ::File.read(@sincedb_path).chomp.strip
+          # If the file was created but we didn't have the time to write to it
+          return content.empty? ? Time.new(0) : Time.parse(content)
+        else
+          return Time.new(0)
+        end
+      end
+
+      def write(since = nil)
+        since = Time.now() if since.nil?
+        ::File.open(@sincedb_path, 'w') { |file| file.write(since.to_s) }
+      end
+    end
+  end
+end # class LogStash::Inputs::S3