logstash-output-oss 0.1.1-java

data/lib/logstash/outputs/oss/file_generator.rb

@@ -0,0 +1,69 @@
+# encoding: utf-8
+
+require 'uuid'
+require 'logstash/outputs/oss/gzip_file'
+require 'logstash/outputs/oss/temporary_file'
+
+module LogStash
+  module Outputs
+    class OSS
+      class FileGenerator
+        FILE_MODE = "a"
+        STRFTIME = "%Y-%m-%dT%H.%M"
+
+        attr_accessor :index, :prefix, :encoding, :temporary_directory, :current_file
+
+        # `prefix`/logstash.oss.{random-uuid}.{%Y-%m-%dT%H.%M}.part-{index}.{extension}
+        def initialize(prefix, encoding, temporary_directory)
+          @index = 0
+          @prefix = prefix
+          # gzip or plain
+          @encoding = encoding
+          # temporary directory to save temporary file before upload to OSS
+          @temporary_directory = temporary_directory
+
+          @lock = Mutex.new
+
+          rotate
+        end
+
+        def rotate
+          @current_file = create_file
+          @index += 1
+          @current_file
+        end
+
+        def with_lock
+          @lock.synchronize do
+            yield self
+          end
+        end
+
+        def extension
+          @encoding == "gzip" ? "gz" : "data"
+        end
+
+        def gzip?
+          @encoding == "gzip"
+        end
+
+        private
+        def create_file
+          uuid = UUID.new.generate
+          file_name = "ls.oss.#{uuid}.#{Time.now.strftime(STRFTIME)}.part-#{index}.#{extension}"
+          object_key = ::File.join(prefix, file_name)
+          local_path = ::File.join(temporary_directory, uuid)
+
+          FileUtils.mkdir_p(::File.join(local_path, prefix))
+          file = if gzip?
+                   GzipFile.new(::File.open(::File.join(local_path, object_key), FILE_MODE))
+                 else
+                   ::File.open(::File.join(local_path, object_key), FILE_MODE)
+                 end
+
+          TemporaryFile.new(file, object_key, local_path)
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/oss/file_manager.rb

@@ -0,0 +1,87 @@
+# encoding: utf-8
+
+require "java"
+require "concurrent"
+require "concurrent/timer_task"
+require "logstash/util"
+require "logstash/outputs/oss/file_generator"
+
+ConcurrentHashMap = java.util.concurrent.ConcurrentHashMap
+
+module LogStash
+  module Outputs
+    class OSS
+      class FileManager
+
+        STALE_FILES_CHECK_INTERVAL_IN_SECONDS = 15 * 60
+
+        def initialize(logger, encoding, temporary_directory)
+          @logger = logger
+          @encoding = encoding
+
+          @temporary_directory = temporary_directory
+          # map of generators
+          # since `prefix` support string interpolation, so we will write many files at the same time
+          @prefixed_generators = ConcurrentHashMap.new
+
+          @file_generator_initialize = FileGeneratorInitializer.new(encoding, temporary_directory)
+
+          start_stale_files_check
+        end
+
+        def get_file_generator(prefix)
+          @prefixed_generators.computeIfAbsent(prefix, @file_generator_initialize).with_lock {|generator| yield generator}
+        end
+
+        def prefixes
+          @prefixed_generators.keySet
+        end
+
+        private
+        def remove_stale_files
+          prefixes.each do |prefix|
+            get_file_generator(prefix) do |file_generator|
+              file = file_generator.current_file
+              if file.size == 0 && Time.now - file.ctime > STALE_FILES_CHECK_INTERVAL_IN_SECONDS
+                @logger.info("Logstash OSS Output Plugin starts to remove stale file",
+                             :key => file.key,
+                             :path => file.path,
+                             :size => file.size,
+                             :thread => Thread.current.to_s)
+                @prefixed_generators.remove(prefix)
+                file.delete!
+              end
+            end
+          end
+        end
+
+        private
+        def start_stale_files_check
+          @stale_check = Concurrent::TimerTask.new(:execution_interval => STALE_FILES_CHECK_INTERVAL_IN_SECONDS) do
+            @logger.info("Logstash OSS Output Plugin: start to check stale files")
+            remove_stale_files
+          end
+
+          @stale_check.execute
+        end
+
+        public
+        def close
+          @stale_check.shutdown
+        end
+
+        class FileGeneratorInitializer
+          include java.util.function.Function
+          def initialize(encoding, temporary_directory)
+            @encoding = encoding
+            @temporary_directory = temporary_directory
+          end
+
+          def apply(prefix)
+            FileGenerator.new(prefix, @encoding, @temporary_directory)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/oss/file_uploader.rb

@@ -0,0 +1,69 @@
+# encoding: utf-8
+require 'java'
+java_import 'com.aliyun.oss.model.ObjectMetadata'
+java_import 'java.io.FileInputStream'
+
+module LogStash
+  module Outputs
+    class OSS
+      class FileUploader
+        TIME_BEFORE_RETRY_SECONDS = 3
+
+        attr_reader :oss, :bucket, :additional_oss_settings, :logger
+
+        def initialize(oss, bucket, additional_oss_settings, logger, thread_pool)
+          @oss = oss
+          @bucket = bucket
+          @additional_oss_settings = additional_oss_settings
+          @logger = logger
+          @thread_pool = thread_pool
+        end
+
+        def upload_async(file, options = {})
+          @thread_pool.post do
+            LogStash::Util.set_thread_name("Logstash OSS Output Plugin: output uploader, file: #{file.path}")
+            upload(file, options)
+          end
+        end
+
+        def upload(file, options = {})
+          meta = ObjectMetadata.new
+          meta.setContentLength(file.size)
+          unless @additional_oss_settings.nil?
+            if @additional_oss_settings.include?(LogStash::Outputs::OSS::SERVER_SIDE_ENCRYPTION_ALGORITHM_KEY)
+              unless @additional_oss_settings[LogStash::Outputs::OSS::SERVER_SIDE_ENCRYPTION_ALGORITHM_KEY].empty?
+                meta.setServerSideEncryption(@additional_oss_settings[LogStash::Outputs::OSS::SERVER_SIDE_ENCRYPTION_ALGORITHM_KEY])
+              end
+            end
+          end
+
+          stream = nil
+          begin
+            stream = FileInputStream.new(file.path)
+            oss.putObject(@bucket, file.key, stream, meta)
+          rescue Errno::ENOENT => e
+            logger.error("Logstash OSS Output Plugin: file to be uploaded doesn't exist!", :exception => e.class, :message => e.message, :path => file.path, :backtrace => e.backtrace)
+          rescue => e
+            @logger.error("Logstash OSS Output Plugin: uploading failed, retrying.", :exception => e.class, :message => e.message, :path => file.path, :backtrace => e.backtrace)
+            sleep TIME_BEFORE_RETRY_SECONDS
+            retry
+          ensure
+            unless stream.nil?
+              stream.close
+            end
+          end
+
+          options[:on_complete].call(file) unless options[:on_complete].nil?
+          rescue => e
+            logger.error("Logstash OSS Output Plugin: an error occurred in the `on_complete` uploader", :exception => e.class, :message => e.message, :path => file.path, :backtrace => e.backtrace)
+            raise e
+        end
+
+        def close
+          @thread_pool.shutdown
+          @thread_pool.wait_for_termination(nil)
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/oss/gzip_file.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+
+module LogStash
+  module Outputs
+    class OSS
+      class GzipFile
+        extend Forwardable
+
+        def_delegators :@gzip_writer, :write, :close
+        attr_reader :file, :gzip_writer
+
+        def initialize(file)
+          @file = file
+          @gzip_writer = Zlib::GzipWriter.new(file)
+        end
+
+        def path
+          @gzip_writer.to_io.path
+        end
+
+        def size
+          if @gzip_writer.pos == 0
+            0
+          else
+            @gzip_writer.flush
+            @gzip_writer.to_io.size
+          end
+        end
+
+        def fsync
+          @gzip_writer.to_io.fsync
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/oss/rotations/hybrid_rotation.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+require "logstash/outputs/oss/rotations/size_based_rotation"
+require "logstash/outputs/oss/rotations/time_based_rotation"
+
+module LogStash
+  module Outputs
+    class OSS
+      class HybridRotation
+        def initialize(size_rotate, time_rotate)
+          @size_strategy = SizeBasedRotation.new(size_rotate)
+          @time_strategy = TimeBasedRotation.new(time_rotate)
+        end
+
+        def rotate?(file)
+          @size_strategy.rotate?(file) || @time_strategy.rotate?(file)
+        end
+
+        def needs_periodic_check?
+          true
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/oss/rotations/size_based_rotation.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+module LogStash
+  module Outputs
+    class OSS
+      class SizeBasedRotation
+        attr_reader :size_rotate
+
+        def initialize(size_rotate)
+          if size_rotate <= 0
+            raise LogStash::ConfigurationError, "Logstash OSS output has wrong configuration: size_rotate must be positive if strategy is `size`"
+          end
+          @size_rotate = size_rotate
+        end
+
+        def rotate?(file)
+          file.size >= @size_rotate
+        end
+
+        def needs_periodic_check?
+          false
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/oss/rotations/time_based_rotation.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+module LogStash
+  module Outputs
+    class OSS
+      class TimeBasedRotation
+        attr_reader :time_rotate
+
+        def initialize(time_rotate)
+          if time_rotate <= 0
+            raise LogStash::ConfigurationError, "Logstash OSS output has wrong configuration: time_rotate must be positive if strategy is `time`"
+          end
+          @time_rotate = time_rotate * 60
+        end
+
+        def rotate?(file)
+          file.size > 0 && (Time.now - file.ctime) >= @time_rotate
+        end
+
+        def needs_periodic_check?
+          true
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/oss/temporary_file.rb

@@ -0,0 +1,59 @@
+# encoding: utf-8
+
+module LogStash
+  module Outputs
+    class OSS
+      class TemporaryFile
+        extend Forwardable
+
+        def_delegators :@file, :path, :write, :close, :fsync
+
+        attr_reader :file
+
+        def initialize(file, object_key, temporary_path)
+          @file = file
+          @object_key = object_key
+          @temporary_path = temporary_path
+          @creation_time = Time.now
+        end
+
+        def ctime
+          @creation_time
+        end
+
+        def temporary_path
+          @temporary_path
+        end
+
+        def size
+          begin
+            @file.size
+          rescue IOError
+            ::File.size(path)
+          end
+        end
+
+        def key
+          @object_key.gsub(/^\//, "")
+        end
+
+        def delete!
+          @file.close rescue IOError
+          FileUtils.rm_r(@temporary_path, :secure => true)
+        end
+
+        def empty?
+          size == 0
+        end
+
+        def self.create_existing_file(path, temporary_directory)
+          # path is #{temporary_directory}/${uuid}/${prefix}/${key}
+          elements = Pathname.new(path).relative_path_from(Pathname.new(temporary_directory)).to_s.split(::File::SEPARATOR)
+          uuid = elements[0]
+          object_key = ::File.join(elements.slice(1, elements.size - 1))
+          TemporaryFile.new(::File.open(path, "r"), object_key, ::File.join(temporary_directory, uuid))
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/oss/version.rb

@@ -0,0 +1,14 @@
+# encoding: utf-8
+
+module LogStash
+  module Outputs
+    class OSS
+      class Version
+        VERSION = "0.1.1"
+        def self.version
+          VERSION
+        end
+      end
+    end
+  end
+end

data/logstash-output-oss.gemspec

@@ -0,0 +1,30 @@
+Gem::Specification.new do |s|
+  s.name          = 'logstash-output-oss'
+  s.version       = '0.1.1'
+  s.licenses      = ['Apache-2.0']
+  s.summary       = 'Sends Logstash events to the Aliyun Object Storage Service'
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gem-name. This gem is not a stand-alone program"
+  s.authors       = ['jinhu.wu']
+  s.email         = 'jinhu.wu.nju@gmail.com'
+  s.require_paths = ['lib']
+  s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+  s.add_runtime_dependency "logstash-codec-plain", "~> 3.0"
+  s.add_runtime_dependency "concurrent-ruby", "~> 1.0"
+  s.add_runtime_dependency "uuid", '~> 2.3', '>= 2.3.9'
+  s.add_development_dependency "logstash-devutils", "~> 1.3"
+  s.add_development_dependency "logstash-codec-line", "~> 3.0"
+  s.platform = 'java'
+  s.add_runtime_dependency 'jar-dependencies', '~> 0.3'
+  s.requirements << 'jar com.aliyun.oss:aliyun-sdk-oss, 3.4.0'
+end