logstash-output-s3 3.2.0 → 4.0.0

data/lib/logstash/outputs/s3/file_repository.rb

@@ -0,0 +1,120 @@
+# encoding: utf-8
+require "java"
+require "concurrent"
+require "concurrent/timer_task"
+require "logstash/util"
+
+ConcurrentHashMap = java.util.concurrent.ConcurrentHashMap
+
+module LogStash
+  module Outputs
+    class S3
+      class FileRepository
+        DEFAULT_STATE_SWEEPER_INTERVAL_SECS = 60
+        DEFAULT_STALE_TIME_SECS = 15 * 60
+        # Ensure that all access or work done
+        # on a factory is threadsafe
+        class PrefixedValue
+          def initialize(file_factory, stale_time)
+            @file_factory = file_factory
+            @lock = Mutex.new
+            @stale_time = stale_time
+          end
+
+          def with_lock
+            @lock.synchronize {
+              yield @file_factory
+            }
+          end
+
+          def stale?
+            with_lock { |factory| factory.current.size == 0 && (Time.now - factory.current.ctime > @stale_time) }
+          end
+
+          def apply(prefix)
+            return self
+          end
+
+          def delete!
+            with_lock{ |factory| factory.current.delete! }
+          end
+        end
+
+        class FactoryInitializer
+          def initialize(tags, encoding, temporary_directory, stale_time)
+            @tags = tags
+            @encoding = encoding
+            @temporary_directory = temporary_directory
+            @stale_time = stale_time
+          end
+
+          def apply(prefix_key)
+            PrefixedValue.new(TemporaryFileFactory.new(prefix_key, @tags, @encoding, @temporary_directory), @stale_time)
+          end
+        end
+
+        def initialize(tags, encoding, temporary_directory,
+                       stale_time = DEFAULT_STALE_TIME_SECS,
+                       sweeper_interval = DEFAULT_STATE_SWEEPER_INTERVAL_SECS)
+          # The path need to contains the prefix so when we start
+          # logtash after a crash we keep the remote structure
+          @prefixed_factories =  ConcurrentHashMap.new
+
+          @sweeper_interval = sweeper_interval
+
+          @factory_initializer = FactoryInitializer.new(tags, encoding, temporary_directory, stale_time)
+
+          start_stale_sweeper
+        end
+
+        def keys
+          @prefixed_factories.keySet
+        end
+
+        def each_files
+          @prefixed_factories.elements.each do |prefixed_file|
+            prefixed_file.with_lock { |factory| yield factory.current }
+          end
+        end
+
+        # Return the file factory
+        def get_factory(prefix_key)
+          @prefixed_factories.computeIfAbsent(prefix_key, @factory_initializer).with_lock { |factory| yield factory }
+        end
+
+        def get_file(prefix_key)
+          get_factory(prefix_key) { |factory| yield factory.current }
+        end
+
+        def shutdown
+          stop_stale_sweeper
+        end
+
+        def size
+          @prefixed_factories.size
+        end
+
+        def remove_stale(k, v)
+          if v.stale?
+            @prefixed_factories.remove(k, v)
+            v.delete!
+          end
+        end
+
+        def start_stale_sweeper
+          @stale_sweeper = Concurrent::TimerTask.new(:execution_interval => @sweeper_interval) do
+            LogStash::Util.set_thread_name("S3, Stale factory sweeper")
+
+            @prefixed_factories.forEach{|k,v| remove_stale(k,v)}
+          end
+
+          @stale_sweeper.execute
+        end
+
+        def stop_stale_sweeper
+          @stale_sweeper.shutdown
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/s3/patch.rb

@@ -0,0 +1,22 @@
+# This is patch related to the autoloading and ruby
+#
+# The fix exist in jruby 9k but not in the current jruby, not sure when or it will be backported
+# https://github.com/jruby/jruby/issues/3645
+#
+# AWS is doing tricky name discovery in the module to generate the correct error class and
+# this strategy is bogus in jruby and `eager_autoload` don't fix this issue.
+#
+# This will be a short lived patch since AWS is removing the need.
+# see: https://github.com/aws/aws-sdk-ruby/issues/1301#issuecomment-261115960
+old_stderr = $stderr
+
+$stderr = StringIO.new
+begin
+  module Aws
+    const_set(:S3, Aws::S3)
+  end
+ensure
+  $stderr = old_stderr
+end
+
+

data/lib/logstash/outputs/s3/path_validator.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+module LogStash
+  module Outputs
+    class S3
+      class PathValidator
+        INVALID_CHARACTERS = "\^`><"
+
+        def self.valid?(name)
+          name.match(matches_re).nil?
+        end
+
+        def self.matches_re
+          /[#{Regexp.escape(INVALID_CHARACTERS)}]/
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/s3/size_and_time_rotation_policy.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+require "logstash/outputs/s3/size_rotation_policy"
+require "logstash/outputs/s3/time_rotation_policy"
+
+module LogStash
+  module Outputs
+    class S3
+      class SizeAndTimeRotationPolicy
+        def initialize(file_size, time_file)
+          @size_strategy = SizeRotationPolicy.new(file_size)
+          @time_strategy = TimeRotationPolicy.new(time_file)
+        end
+
+        def rotate?(file)
+          @size_strategy.rotate?(file) || @time_strategy.rotate?(file)
+        end 
+
+        def needs_periodic?
+          true
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/s3/size_rotation_policy.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+module LogStash
+  module Outputs
+    class S3
+      class SizeRotationPolicy
+        attr_reader :size_file
+
+        def initialize(size_file)
+          if size_file <= 0
+            raise LogStash::ConfigurationError, "`size_file` need to be greather than 0"
+          end
+
+          @size_file = size_file
+        end
+
+        def rotate?(file)
+          file.size >= size_file
+        end
+
+        def needs_periodic?
+          false
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/s3/temporary_file.rb

@@ -0,0 +1,71 @@
+# encoding: utf-8
+require "thread"
+require "forwardable"
+require "fileutils"
+
+module LogStash
+  module Outputs
+    class S3
+      # Wrap the actual file descriptor into an utility classe
+      # It make it more OOP and easier to reason with the paths.
+      class TemporaryFile
+        extend Forwardable
+
+        def_delegators :@fd, :path, :write, :close, :fsync
+
+        attr_reader :fd
+
+        def initialize(key, fd, temp_path)
+          @fd = fd
+          @key = key
+          @temp_path = temp_path
+          @created_at = Time.now
+        end
+
+        def ctime
+          @created_at
+        end
+
+        def temp_path
+          @temp_path
+        end
+
+        def size
+          # Use the fd size to get the accurate result,
+          # so we dont have to deal with fsync
+          # if the file is close we will use the File::size
+          begin
+            @fd.size
+          rescue IOError
+            ::File.size(path)
+          end
+        end
+
+        def key
+          @key.gsub(/^\//, "")
+        end
+
+        # Each temporary file is made inside a directory named with an UUID,
+        # instead of deleting the file directly and having the risk of deleting other files
+        # we delete the root of the UUID, using a UUID also remove the risk of deleting unwanted file, it acts as
+        # a sandbox.
+        def delete!
+          @fd.close
+          ::FileUtils.rm_rf(@temp_path, :secure => true)
+        end
+
+        def empty?
+          size == 0
+        end
+
+        def self.create_from_existing_file(file_path, temporary_folder)
+          key_parts = Pathname.new(file_path).relative_path_from(temporary_folder).to_s.split(::File::SEPARATOR)
+
+          TemporaryFile.new(key_parts.slice(1, key_parts.size).join("/"),
+                         ::File.open(file_path, "r"),
+                         ::File.join(temporary_folder, key_parts.slice(0, 1)))
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/s3/temporary_file_factory.rb

@@ -0,0 +1,123 @@
+# encoding: utf-8
+require "socket"
+require "securerandom"
+require "fileutils"
+require "zlib"
+require "forwardable"
+
+module LogStash
+  module Outputs
+    class S3
+      # Since the file can contains dynamic part, we have to handle a more local structure to
+      # allow a nice recovery from a crash.
+      #
+      # The local structure will look like this.
+      #
+      # <TEMPORARY_PATH>/<UUID>/<prefix>/ls.s3.localhost.%Y-%m-%dT%H.%m.tag_es_fb.part1.txt.gz
+      #
+      # Since the UUID should be fairly unique I can destroy the whole path when an upload is complete.
+      # I do not have to mess around to check if the other directory have file in it before destroying them.
+      class TemporaryFileFactory
+        FILE_MODE = "a"
+        GZIP_ENCODING = "gzip"
+        GZIP_EXTENSION = "txt.gz"
+        TXT_EXTENSION = "txt"
+        STRFTIME = "%Y-%m-%dT%H.%M"
+
+        attr_accessor :counter, :tags, :prefix, :encoding, :temporary_directory, :current
+
+        def initialize(prefix, tags, encoding, temporary_directory)
+          @counter = 0
+          @prefix = prefix
+
+          @tags = tags
+          @encoding = encoding
+          @temporary_directory = temporary_directory
+          @lock = Mutex.new
+
+          rotate!
+        end
+
+        def rotate!
+          @lock.synchronize {
+            @current = new_file
+            increment_counter
+            @current
+          }
+        end
+
+        private
+        def extension
+          gzip? ? GZIP_EXTENSION : TXT_EXTENSION
+        end
+
+        def gzip?
+          encoding == GZIP_ENCODING
+        end
+
+        def increment_counter
+          @counter += 1
+        end
+
+        def current_time
+          Time.now.strftime(STRFTIME)
+        end
+
+        def generate_name
+          filename = "ls.s3.#{SecureRandom.uuid}.#{current_time}"
+
+          if tags.size > 0
+            "#{filename}.tag_#{tags.join('.')}.part#{counter}.#{extension}"
+          else
+            "#{filename}.part#{counter}.#{extension}"
+          end
+        end
+
+        def new_file
+          uuid = SecureRandom.uuid
+          name = generate_name
+          path = ::File.join(temporary_directory, uuid)
+          key = ::File.join(prefix, name)
+
+          FileUtils.mkdir_p(::File.join(path, prefix))
+
+          io = if gzip?
+                 # We have to use this wrapper because we cannot access the size of the
+                 # file directly on the gzip writer.
+                 IOWrappedGzip.new(::File.open(::File.join(path, key), FILE_MODE))
+               else
+                 ::File.open(::File.join(path, key), FILE_MODE)
+               end
+
+          TemporaryFile.new(key, io, path)
+        end
+
+        class IOWrappedGzip
+          extend Forwardable
+
+          def_delegators :@gzip_writer, :write, :close
+          attr_reader :file_io, :gzip_writer
+
+          def initialize(file_io)
+            @file_io = file_io
+            @gzip_writer = Zlib::GzipWriter.open(file_io)
+          end
+
+          def path
+            @gzip_writer.to_io.path
+          end
+
+          def size
+            # to get the current file size
+            @gzip_writer.flush
+            @gzip_writer.to_io.size
+          end
+
+          def fsync
+            @gzip_writer.to_io.fsync
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/s3/time_rotation_policy.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+module LogStash
+  module Outputs
+    class S3
+      class TimeRotationPolicy
+        attr_reader :time_file
+
+        def initialize(time_file)
+          if time_file <= 0
+            raise LogStash::ConfigurationError, "`time_file` need to be greather than 0"
+          end
+
+          @time_file = time_file * 60
+        end
+
+        def rotate?(file)
+          file.size > 0 && (Time.now - file.ctime) >= time_file
+        end
+
+        def needs_periodic?
+          true
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/s3/uploader.rb

@@ -0,0 +1,59 @@
+# encoding: utf-8
+require "logstash/util"
+require "aws-sdk"
+
+module LogStash
+  module Outputs
+    class S3
+      class Uploader
+        TIME_BEFORE_RETRYING_SECONDS = 1
+        DEFAULT_THREADPOOL = Concurrent::ThreadPoolExecutor.new({
+                                                                  :min_threads => 1,
+                                                                  :max_threads => 8,
+                                                                  :max_queue => 1,
+                                                                  :fallback_policy => :caller_runs
+                                                                })
+
+
+        attr_reader :bucket, :upload_options, :logger
+
+        def initialize(bucket, logger, threadpool = DEFAULT_THREADPOOL)
+          @bucket = bucket
+          @workers_pool = threadpool
+          @logger = logger
+        end
+
+        def upload_async(file, options = {})
+          @workers_pool.post do
+            LogStash::Util.set_thread_name("S3 output uploader, file: #{file.path}")
+            upload(file, options)
+          end
+        end
+
+        def upload(file, options = {})
+          upload_options = options.fetch(:upload_options, {})
+
+          begin
+            obj = bucket.object(file.key)
+            obj.upload_file(file.path, upload_options)
+          rescue => e
+            # When we get here it usually mean that S3 tried to do some retry by himself (default is 3)
+            # When the retry limit is reached or another error happen we will wait and retry.
+            #
+            # Thread might be stuck here, but I think its better than losing anything
+            # its either a transient errors or something bad really happened.
+            logger.error("Uploading failed, retrying", :exception => e, :path => file.path, :backtrace => e.backtrace)
+            retry
+          end
+
+          options[:on_complete].call(file) unless options[:on_complete].nil?
+        end
+
+        def stop
+          @workers_pool.shutdown
+          @workers_pool.wait_for_termination(nil) # block until its done
+        end
+      end
+    end
+  end
+end