logstash-output-s3-zst 1.0.0

data/lib/logstash/outputs/s3/size_rotation_policy.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+module LogStash
+  module Outputs
+    class S3
+      class SizeRotationPolicy
+        attr_reader :size_file
+
+        def initialize(size_file)
+          if size_file <= 0
+            raise LogStash::ConfigurationError, "`size_file` need to be greater than 0"
+          end
+
+          @size_file = size_file
+        end
+
+        def rotate?(file)
+          file.size >= size_file
+        end
+
+        def needs_periodic?
+          false
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/s3/temporary_file.rb

@@ -0,0 +1,114 @@
+# encoding: utf-8
+require "thread"
+require "forwardable"
+require "fileutils"
+require "logstash-output-s3_jars"
+
+module LogStash
+  module Outputs
+    class S3
+
+      java_import 'org.logstash.plugins.outputs.s3.GzipUtil'
+
+      # Wrap the actual file descriptor into an utility class
+      # Make it more OOP and easier to reason with the paths.
+      class TemporaryFile
+        extend Forwardable
+
+        GZIP_EXTENSION = "txt.gz"
+        TXT_EXTENSION = "txt"
+        RECOVERED_FILE_NAME_TAG = "-recovered"
+
+        def_delegators :@fd, :path, :write, :close, :fsync
+
+        attr_reader :fd
+
+        def initialize(key, fd, temp_path)
+          @fd = fd
+          @key = key
+          @temp_path = temp_path
+          @created_at = Time.now
+        end
+
+        def ctime
+          @created_at
+        end
+
+        def temp_path
+          @temp_path
+        end
+
+        def size
+          # Use the fd size to get the accurate result,
+          # so we dont have to deal with fsync
+          # if the file is close, fd.size raises an IO exception so we use the File::size
+          begin
+            # fd is nil when LS tries to recover gzip file but fails
+            return 0 unless @fd != nil
+            @fd.size
+          rescue IOError
+            ::File.size(path)
+          end
+        end
+
+        def key
+          @key.gsub(/^\//, "")
+        end
+
+        # Each temporary file is created inside a directory named with an UUID,
+        # instead of deleting the file directly and having the risk of deleting other files
+        # we delete the root of the UUID, using a UUID also remove the risk of deleting unwanted file, it acts as
+        # a sandbox.
+        def delete!
+          @fd.close rescue IOError # force close anyway
+          FileUtils.rm_r(@temp_path, :secure => true)
+        end
+
+        def empty?
+          size == 0
+        end
+
+        # only to cover the case where LS cannot restore corrupted file, file is not exist
+        def recoverable?
+          !@fd.nil?
+        end
+
+        def self.create_from_existing_file(file_path, temporary_folder)
+          key_parts = Pathname.new(file_path).relative_path_from(temporary_folder).to_s.split(::File::SEPARATOR)
+
+          # recover gzip file and compress back before uploading to S3
+          if file_path.end_with?("." + GZIP_EXTENSION)
+            file_path = self.recover(file_path)
+          end
+          TemporaryFile.new(key_parts.slice(1, key_parts.size).join("/"),
+                         ::File.exist?(file_path) ? ::File.open(file_path, "r") : nil, # for the nil case, file size will be 0 and upload will be ignored.
+                         ::File.join(temporary_folder, key_parts.slice(0, 1)))
+        end
+
+        def self.gzip_extension
+          GZIP_EXTENSION
+        end
+
+        def self.text_extension
+          TXT_EXTENSION
+        end
+
+        def self.recovery_file_name_tag
+          RECOVERED_FILE_NAME_TAG
+        end
+
+        private
+        def self.recover(file_path)
+          full_gzip_extension = "." + GZIP_EXTENSION
+          recovered_txt_file_path = file_path.gsub(full_gzip_extension, RECOVERED_FILE_NAME_TAG + "." + TXT_EXTENSION)
+          recovered_gzip_file_path = file_path.gsub(full_gzip_extension, RECOVERED_FILE_NAME_TAG + full_gzip_extension)
+          GzipUtil.recover(file_path, recovered_txt_file_path)
+          if ::File.exist?(recovered_txt_file_path) && !::File.zero?(recovered_txt_file_path)
+            GzipUtil.compress(recovered_txt_file_path, recovered_gzip_file_path)
+          end
+          recovered_gzip_file_path
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/s3/temporary_file_factory.rb

@@ -0,0 +1,126 @@
+# encoding: utf-8
+require "socket"
+require "securerandom"
+require "fileutils"
+require "zlib"
+require "forwardable"
+
+module LogStash
+  module Outputs
+    class S3
+      # Since the file can contains dynamic part, we have to handle a more local structure to
+      # allow a nice recovery from a crash.
+      #
+      # The local structure will look like this.
+      #
+      # <TEMPORARY_PATH>/<UUID>/<prefix>/ls.s3.localhost.%Y-%m-%dT%H.%m.tag_es_fb.part1.txt.gz
+      #
+      # Since the UUID should be fairly unique I can destroy the whole path when an upload is complete.
+      # I do not have to mess around to check if the other directory have file in it before destroying them.
+      class TemporaryFileFactory
+        FILE_MODE = "a"
+        STRFTIME = "%Y-%m-%dT%H.%M"
+
+        attr_accessor :counter, :tags, :prefix, :encoding, :temporary_directory, :current
+
+        def initialize(prefix, tags, encoding, temporary_directory)
+          @counter = 0
+          @prefix = prefix
+
+          @tags = tags
+          @encoding = encoding
+          @temporary_directory = temporary_directory
+          @lock = Mutex.new
+
+          rotate!
+        end
+
+        def rotate!
+          @lock.synchronize {
+            @current = new_file
+            increment_counter
+            @current
+          }
+        end
+
+        private
+        def extension
+          gzip? ? TemporaryFile.gzip_extension : TemporaryFile.text_extension
+        end
+
+        def gzip?
+          encoding == GZIP_ENCODING
+        end
+
+        def increment_counter
+          @counter += 1
+        end
+
+        def current_time
+          Time.now.strftime(STRFTIME)
+        end
+
+        def generate_name
+          filename = "ls.s3.#{SecureRandom.uuid}.#{current_time}"
+
+          if tags.size > 0
+            "#{filename}.tag_#{tags.join('.')}.part#{counter}.#{extension}"
+          else
+            "#{filename}.part#{counter}.#{extension}"
+          end
+        end
+
+        def new_file
+          uuid = SecureRandom.uuid
+          name = generate_name
+          path = ::File.join(temporary_directory, uuid)
+          key = ::File.join(prefix, name)
+
+          FileUtils.mkdir_p(::File.join(path, prefix))
+
+          io = if gzip?
+                 # We have to use this wrapper because we cannot access the size of the
+                 # file directly on the gzip writer.
+                 IOWrappedGzip.new(::File.open(::File.join(path, key), FILE_MODE))
+               else
+                 ::File.open(::File.join(path, key), FILE_MODE)
+               end
+
+          TemporaryFile.new(key, io, path)
+        end
+
+        class IOWrappedGzip
+          extend Forwardable
+
+          def_delegators :@gzip_writer, :write, :close
+          attr_reader :file_io, :gzip_writer
+
+          def initialize(file_io)
+            @file_io = file_io
+            @gzip_writer = Zlib::GzipWriter.new(file_io)
+          end
+
+          def path
+            @gzip_writer.to_io.path
+          end
+
+          def size
+            # to get the current file size
+            if @gzip_writer.pos == 0
+              # Ensure a zero file size is returned when nothing has
+              # yet been written to the gzip file.
+              0
+            else
+              @gzip_writer.flush
+              @gzip_writer.to_io.size
+            end
+          end
+
+          def fsync
+            @gzip_writer.to_io.fsync
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/s3/time_rotation_policy.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+module LogStash
+  module Outputs
+    class S3
+      class TimeRotationPolicy
+        attr_reader :time_file
+
+        def initialize(time_file)
+          if time_file <= 0
+            raise LogStash::ConfigurationError, "`time_file` need to be greather than 0"
+          end
+
+          @time_file = time_file * 60
+        end
+
+        def rotate?(file)
+          file.size > 0 && (Time.now - file.ctime) >= time_file
+        end
+
+        def needs_periodic?
+          true
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/s3/uploader.rb

@@ -0,0 +1,82 @@
+# encoding: utf-8
+require "logstash/util"
+require "aws-sdk"
+require 'securerandom'
+require 'zstd'
+
+module LogStash
+  module Outputs
+    class S3
+      class Uploader
+
+        DEFAULT_THREADPOOL = Concurrent::ThreadPoolExecutor.new({
+                                                                  :min_threads => 1,
+                                                                  :max_threads => 8,
+                                                                  :max_queue => 1,
+                                                                  :fallback_policy => :caller_runs
+                                                                })
+
+        attr_reader :bucket, :upload_options, :logger
+
+        def initialize(bucket, logger, threadpool = DEFAULT_THREADPOOL, retry_count: Float::INFINITY, retry_delay: 1)
+          @bucket = bucket
+          @workers_pool = threadpool
+          @logger = logger
+          @retry_count = retry_count
+          @retry_delay = retry_delay
+        end
+
+        def upload_async(file, options = {})
+          @workers_pool.post do
+            LogStash::Util.set_thread_name("S3 output uploader, file: #{file.path}")
+            upload(file, options)
+          end
+        end
+
+        # uploads a TemporaryFile to S3
+        def upload(file, options = {})
+          upload_options = options.fetch(:upload_options, {})
+
+          zstd_compressed_file = "#{SecureRandom.uuid}.json.zst"
+
+          compressed = Zstd.compress_file(file.path, zstd_compressed_tempfile)
+
+          tries = 0
+          begin
+            obj = bucket.object(compressed.key)
+            obj.upload_file(compressed.path, upload_options)
+          rescue Errno::ENOENT => e
+            logger.error("File doesn't exist! Unrecoverable error.", :exception => e.class, :message => e.message, :path => file.path, :backtrace => e.backtrace)
+          rescue => e
+            # When we get here it usually mean that S3 tried to do some retry by himself (default is 3)
+            # When the retry limit is reached or another error happen we will wait and retry.
+            #
+            # Thread might be stuck here, but I think its better than losing anything
+            # its either a transient errors or something bad really happened.
+            if tries < @retry_count
+              tries += 1
+              logger.warn("Uploading failed, retrying (##{tries} of #{@retry_count})", :exception => e.class, :message => e.message, :path => file.path, :backtrace => e.backtrace)
+              sleep @retry_delay
+              retry
+            else
+              logger.error("Failed to upload file (retried #{@retry_count} times).", :exception => e.class, :message => e.message, :path => file.path, :backtrace => e.backtrace)
+            end
+          end
+
+          begin
+            options[:on_complete].call(compressed) unless options[:on_complete].nil?
+          rescue => e
+            logger.error("An error occurred in the `on_complete` uploader", :exception => e.class, :message => e.message, :path => compressed.path, :backtrace => e.backtrace)
+            raise e # reraise it since we don't deal with it now
+          end
+        end
+
+        def stop
+          @workers_pool.shutdown
+          @workers_pool.wait_for_termination(nil) # block until its done
+        end
+
+      end
+    end
+  end
+end

data/lib/logstash/outputs/s3/writable_directory_validator.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+module LogStash
+  module Outputs
+    class S3
+      class WritableDirectoryValidator
+        def self.valid?(path)
+          begin
+            FileUtils.mkdir_p(path) unless Dir.exist?(path)
+            ::File.writable?(path)
+          rescue
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/s3/write_bucket_permission_validator.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+require "stud/temporary"
+require "socket"
+require "fileutils"
+
+module LogStash
+  module Outputs
+    class S3
+      class WriteBucketPermissionValidator
+        attr_reader :logger
+
+        def initialize(logger)
+          @logger = logger
+        end
+
+        def valid?(bucket_resource, upload_options = {})
+          begin
+            upload_test_file(bucket_resource, upload_options)
+            true
+          rescue StandardError => e
+            logger.error("Error validating bucket write permissions!",
+              :message => e.message,
+              :class => e.class.name,
+              :backtrace => e.backtrace
+              )
+            false
+          end
+        end
+
+        private
+        def upload_test_file(bucket_resource, upload_options = {})
+          generated_at = Time.now
+
+          key = "logstash-programmatic-access-test-object-#{generated_at}"
+          content = "Logstash permission check on #{generated_at}, by #{Socket.gethostname}"
+
+          begin
+            f = Stud::Temporary.file
+            f.write(content)
+            f.fsync
+
+            obj = bucket_resource.object(key)
+            obj.upload_file(f, upload_options)
+
+            begin
+              obj.delete
+            rescue
+              # Try to remove the files on the remote bucket,
+              # but don't raise any errors if that doesn't work.
+              # since we only really need `putobject`.
+            end
+          ensure
+            f.close
+            FileUtils.rm_rf(f.path)
+          end
+        end
+      end
+    end
+  end
+end