logstash-output-qingstor 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7edeee9f910f1ff39de1856a36166c30ad66052e
-  data.tar.gz: 31b21ceee52a6402f89fa782f368450e1f84ebd4
+  metadata.gz: d1fb6b25a4999d3628887cb326a791afddb002dd
+  data.tar.gz: d72aca418dc34fe47c11f303c7f48e459a35715c
 SHA512:
-  metadata.gz: 088f7febcb4b81f11d956c52dc5aa100e01eb7cd4d50670952b9f24ac2b54f24447b17a74900697d23cea1b5b86b42e82972be48b4b92ada16e19b081c7e2e41
-  data.tar.gz: 4467d04227d805798d58f7806ae6f0f8a59cd7b81070126ecb9462104798474d0f508fd705dead75fa86c87607aa2a7c784224fd4270fb75c27ef3fdb77ffd2d
+  metadata.gz: 176d12530d09b496dd051ea70b8694ceda11307b301265114fb64aa46c76564b7a82304489508e37ea0adfedc95656badf0ba0d24aa74236fb06dee5a221835d
+  data.tar.gz: 4ba9d2490ce1564cf202803346fa8661b83e960743161113a9d3248c89e3d618792d57558c3397d2f224470b17ff5a39805344b80eb40ea306825f75a963ea98

data/Gemfile

@@ -1,3 +1,2 @@
 source 'https://rubygems.org'
 gemspec
-

data/README.md

@@ -24,7 +24,7 @@ output {
 
 ```
 
-More configuration details please refer to [code part](lib/logstash/outputs/qingstor.rb).
+More configuration details please refer to [common options](/docs/index.asciidoc).
 
 ## 2. Running your unpublished Plugin in Logstash
 

data/lib/logstash/outputs/qingstor/file_repository.rb

@@ -1,12 +1,14 @@
 # encoding: utf-8
-require "java"
-require "concurrent"
-require "concurrent/timer_task"
-require "logstash/util"
+
+require 'logstash/util'
+require 'logstash/outputs/qingstor'
+require 'java'
+require 'concurrent'
+require 'concurrent/timer_task'
 
 ConcurrentHashMap = java.util.concurrent.ConcurrentHashMap
 
-module LogStash 
+module LogStash
   module Outputs
     class Qingstor
       class FileRepository
@@ -20,22 +22,25 @@ module LogStash
           end
 
           def with_lock
-            @lock.synchronize {
+            @lock.synchronize do
               yield @file_factory
-            }          
-          end 
+            end
+          end
 
           def stale?
-            with_lock { |factory| factory.current.size == 0 && (Time.now - factory.current.ctime > @stale_time) }
-          end 
+            with_lock do |factory|
+              factory.current.empty? &&
+                (Time.now - factory.current.ctime > @stale_time)
+            end
+          end
 
-          def apply(prefix)
-            return self
-          end 
+          def apply(_prefix)
+            self
+          end
 
-          def delete! 
+          def delete!
             with_lock { |factory| factory.current.delete! }
-          end 
+          end
         end
 
         class FactoryInitializer
@@ -44,69 +49,79 @@ module LogStash
             @encoding = encoding
             @temporary_directory = temporary_directory
             @stale_time = stale_time
-          end 
+          end
 
           def apply(prefix_key)
-            PrefixedValue.new(TemporaryFileFactory.new(prefix_key, @tags, @encoding, @temporary_directory), @stale_time) 
-          end 
-        end 
+            PrefixedValue.new(
+              TemporaryFileFactory.new(prefix_key, @tags, @encoding,
+                                       @temporary_directory),
+              @stale_time
+            )
+          end
+        end
 
         def initialize(tags, encoding, temporary_directory,
                        stale_time = DEFAULT_STATE_TIME_SECS,
                        sweeper_interval = DEFAULT_STATE_SWEEPER_INTERVAL_SECS)
-          @prefixed_factories = ConcurrentHashMap.new 
+          @prefixed_factories = ConcurrentHashMap.new
           @sweeper_interval = sweeper_interval
-          @factoryinitializer = FactoryInitializer.new(tags, encoding, temporary_directory, stale_time)
+          @factoryinitializer = FactoryInitializer.new(tags,
+                                                       encoding,
+                                                       temporary_directory,
+                                                       stale_time)
 
           start_stale_sweeper
         end
 
         def keys
           @prefixed_factories.keySet
-        end 
+        end
 
-        def each_files 
+        def each_files
           @prefixed_factories.elements.each do |prefixed_file|
             prefixed_file.with_lock { |factory| yield factory.current }
-          end 
-        end 
+          end
+        end
 
         def get_factory(prefix_key)
-          @prefixed_factories.computeIfAbsent(prefix_key, @factoryinitializer).with_lock { |factory| yield factory }
-        end 
+          @prefixed_factories.computeIfAbsent(prefix_key, @factoryinitializer)
+                             .with_lock { |factory| yield factory }
+        end
 
         def get_file(prefix_key)
           get_factory(prefix_key) { |factory| yield factory.current }
-        end 
+        end
 
-        def shutdown 
-          stop_stale_sweeper 
-        end 
+        def shutdown
+          stop_stale_sweeper
+        end
+
+        def size
+          @prefixed_factories.size
+        end
 
-        def size 
-          @prefixed_factories.size 
-        end 
-        
         def remove_stale(k, v)
-          if v.stale? 
+          if v.stale?
             @prefixed_factories.remove(k, v)
             v.delete!
-          end 
-        end 
+          end
+        end
 
-        def start_stale_sweeper 
-          @stale_sweeper = Concurrent::TimerTask.new(:execution_interval => @sweeper_interval) do 
-            LogStash::Util.set_thread_name("Qingstor, stale factory sweeper")
+        def start_stale_sweeper
+          @stale_sweeper = Concurrent::TimerTask.new(
+            :execution_interval => @sweeper_interval
+          ) do
+            LogStash::Util.set_thread_name('Qingstor, stale factory sweeper')
             @prefixed_factories.forEach { |k, v| remove_stale(k, v) }
-          end 
+          end
 
           @stale_sweeper.execute
-        end 
+        end
 
-        def stop_stale_sweeper 
+        def stop_stale_sweeper
           @stale_sweeper.shutdown
-        end 
-      end 
-    end 
-  end 
-end 
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/qingstor/qingstor_validator.rb

@@ -1,30 +1,31 @@
 # encoding: utf-8
-require "qingstor/sdk"
-require "fileutils"
+
+require 'logstash/outputs/qingstor'
+require 'qingstor/sdk'
 
 module LogStash
   module Outputs
     class Qingstor
-      class QingstorValidator
-
+      module QingstorValidator
         def self.bucket_valid?(bucket)
           res = bucket.head
-          case res[:status_code] 
+          case res[:status_code]
           when 401
-            raise LogStash::ConfigurationError, "Incorrect key id or access key."
+            raise LogStash::ConfigurationError, 'Incorrect key id/access key.'
           when 404
-            raise LogStash::ConfigurationError, "Incorrect bucket/region name."
-          end 
-          true 
-        end 
+            raise LogStash::ConfigurationError, 'Incorrect bucket/region name.'
+          end
+          true
+        end
 
         def self.prefix_valid?(prefix)
-          if prefix.start_with?("/") || prefix.length >= 1024
-            raise LogStash::ConfigurationError, "Prefix must not start with '/' with length less than 1024 "
-          end 
+          if prefix.start_with?('/') || prefix.length >= 1024
+            raise LogStash::ConfigurationError, 'Prefix must not start with'\
+              + " '/' and with length less than 1024 "
+          end
           true
-        end 
-      end 
-    end 
-  end 
-end 
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/qingstor/rotation_policy.rb

@@ -1,71 +1,78 @@
 # encoding: utf-8
-module LogStash 
-  module Outputs 
+
+require 'logstash/outputs/qingstor'
+
+module LogStash
+  module Outputs
     class Qingstor
       class RotationPolicy
-        
         def initialize(policy, file_size, file_time)
           @policy = policy
           case policy
-          when "time"
+          when 'time'
             init_time(file_time)
-          when "size"
+          when 'size'
             init_size(file_size)
-          when "size_and_time"
+          when 'size_and_time'
             init_size(file_size)
-            init_time(file_time)     
-          end 
-        end 
+            init_time(file_time)
+          end
+        end
 
         def init_size(file_size)
           if file_size <= 0
-            raise LogStash::ConfigurationError, "'file_size' need to be greater than 0" 
+            raise LogStash::ConfigurationError, "'file_size' need to "\
+              + 'be greater than 0'
           end
           @file_size = file_size
-        end 
+        end
 
         def init_time(file_time)
           if file_time <= 0
-            raise LogStash::ConfigurationError, "'file_time' need to be greater than 0" 
-          end 
+            raise LogStash::ConfigurationError, "'file_time' need to "\
+              + 'be greater than 0'
+          end
           @file_time = file_time
-        end 
+        end
 
         def rotate?(file)
           case @policy
-          when "time"
+          when 'time'
             time_rotate?(file)
-          when "size"
+          when 'size'
             size_rotate?(file)
-          when "size_and_time"
+          when 'size_and_time'
             size_and_time_rotate?(file)
           end
-        end 
+        end
 
         def size_and_time_rotate?(file)
           size_rotate?(file) || time_rotate?(file)
-        end 
+        end
 
         def size_rotate?(file)
           file.size >= @file_size
-        end 
+        end
 
         def time_rotate?(file)
-          file.size > 0 && (Time.now - file.ctime) >= @file_time
-        end 
+          !file.empty? && (Time.now - file.ctime) >= @file_time
+        end
 
         def needs_periodic?
           case @policy
-          when "time" then
+          when 'time' then
             true
-          when "size_and_time" then
+          when 'size_and_time' then
             true
           else
             false
           end
         end
 
-      end 
-    end 
-  end 
-end 
+        def to_s
+          "#{@policy} #{@file_time} #{@file_size}"
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/qingstor/temporary_file.rb

@@ -1,8 +1,10 @@
 # encoding: utf-8
-require "thread"
-require "forwardable"
-require "fileutils"
-require "pathname"
+
+require 'logstash/outputs/qingstor'
+require 'thread'
+require 'forwardable'
+require 'fileutils'
+require 'pathname'
 
 module LogStash
   module Outputs
@@ -15,49 +17,49 @@ module LogStash
         attr_reader :fd
 
         def initialize(key, fd, tmp_path)
-          @key = key 
+          @key = key
           @fd = fd
           @tmp_path = tmp_path
           @created_at = Time.now
-        end 
+        end
 
-        def ctime 
-          @created_at 
-        end 
-        
-        def tmp_path 
-          @tmp_path 
-        end 
+        def ctime
+          @created_at
+        end
 
-        def size 
-          begin
-            @fd.size
-          rescue IOError
-            ::File.size(path)
-          end 
-        end 
+        attr_reader :tmp_path
 
-        def key 
-          @key.gsub(/^\//, "")
-        end 
+        def size
+          @fd.size
+        rescue IOError
+          ::File.size(path)
+        end
+
+        def key
+          @key.gsub(/^\//, '')
+        end
 
         def delete!
-          @fd.close rescue IOError 
+          begin
+            @fd.close
+          rescue
+            IOError
+          end
           FileUtils.rm_r(@tmp_path, :secure => true)
-        end 
+        end
 
         def empty?
-          size == 0
-        end 
+          size.zero?
+        end
 
         def self.create_from_existing_file(file_path, tmp_folder)
-          key_parts = Pathname.new(file_path).relative_path_from(tmp_folder).to_s.split(::File::SEPARATOR)
-          TemporaryFile.new(key_parts.slice(1, key_parts.size).join("/"),
-                            ::File.open(file_path, "r"),
+          key_parts = Pathname.new(file_path).relative_path_from(tmp_folder)
+                              .to_s.split(::File::SEPARATOR)
+          TemporaryFile.new(key_parts.slice(1, key_parts.size).join('/'),
+                            ::File.open(file_path, 'r'),
                             ::File.join(tmp_folder, key_parts.slice(0, 1)))
-                    
-        end 
-      end 
-    end 
-  end 
-end 
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/qingstor/temporary_file_factory.rb

@@ -1,19 +1,21 @@
 # encoding: utf-8
-require "socket"
-require "securerandom"
-require "fileutils"
-require "zlib"
-require "forwardable"
+
+require 'logstash/outputs/qingstor'
+require 'socket'
+require 'securerandom'
+require 'fileutils'
+require 'zlib'
+require 'forwardable'
 
 module LogStash
   module Outputs
     class Qingstor
       class TemporaryFileFactory
-        FILE_MODE = "a"
-        GZIP_ENCODING = "gzip"
-        GZIP_EXTENSION = "log.gz"
-        TXT_EXTENSION = "log"
-        STRFTIME = "%Y-%m-%dT%H.%M"
+        FILE_MODE = 'a'.freeze
+        GZIP_ENCODING = 'gzip'.freeze
+        GZIP_EXTENSION = 'log.gz'.freeze
+        TXT_EXTENSION = 'log'.freeze
+        STRFTIME = '%Y-%m-%dT%H.%M'.freeze
 
         attr_accessor :counter, :tags, :prefix, :encoding, :tmpdir, :current
 
@@ -23,88 +25,90 @@ module LogStash
           @tags = tags
           @encoding = encoding
           @tmpdir = tmpdir
-          @lock = Mutex.new 
+          @lock = Mutex.new
 
           rotate!
-        end 
+        end
 
         def rotate!
-          @lock.synchronize {
+          @lock.synchronize do
             @current = new_file
             increment_counter
-            @current 
-          }
-        end 
+            @current
+          end
+        end
+
+        private
 
-        private 
         def extension
           gzip? ? GZIP_EXTENSION : TXT_EXTENSION
-        end 
-        
+        end
+
         def gzip?
           encoding == GZIP_ENCODING
-        end 
+        end
 
         def increment_counter
           @counter += 1
-        end 
+        end
 
-        def current_time 
+        def current_time
           Time.new.strftime(STRFTIME)
-        end 
+        end
 
-        def generate_name 
+        def generate_name
           filename = "ls.qingstor.#{SecureRandom.uuid}.#{current_time}"
 
-          if tags.size > 0
+          if !tags.empty?
             "#{filename}.tag_#{tags.join('.')}.part#{counter}.#{extension}"
-          else 
+          else
             "#{filename}.part#{counter}.#{extension}"
-          end 
-        end 
+          end
+        end
 
-        def new_file 
-          uuid = SecureRandom.uuid 
-          name = generate_name 
+        def new_file
+          uuid = SecureRandom.uuid
+          name = generate_name
           path = ::File.join(@tmpdir, uuid)
           key = ::File.join(@prefix, name)
 
           FileUtils.mkdir_p(::File.join(path, @prefix))
 
-          io = if gzip? 
-                 IOWrappedGzip.new(::File.open(::File.join(path, key), FILE_MODE))
-               else 
+          io = if gzip?
+                 IOWrappedGzip.new(::File.open(::File.join(path, key),
+                                               FILE_MODE))
+               else
                  ::File.open(::File.join(path, key), FILE_MODE)
-               end 
+               end
 
           TemporaryFile.new(key, io, path)
         end
 
-        class IOWrappedGzip 
-          extend Forwardable 
+        class IOWrappedGzip
+          extend Forwardable
 
           def_delegators :@gzip_writer, :write, :close
           attr_accessor :file_io, :gzip_writer
-        
+
           def initialize(file_io)
             @file_io = file_io
             @gzip_writer = Zlib::GzipWriter.open(file_io)
-          end 
-
-          def path 
-            @gzip_writer.to_io.path 
-          end 
-          
-          def size 
-            @gzip_writer.flush 
-            @gzip_writer.to_io.size 
-          end 
-
-          def fsync 
-            @gzip_writer.to_io.fsync 
-          end 
-        end 
-      end 
-    end 
-  end 
-end 
+          end
+
+          def path
+            @gzip_writer.to_io.path
+          end
+
+          def size
+            @gzip_writer.flush
+            @gzip_writer.to_io.size
+          end
+
+          def fsync
+            @gzip_writer.to_io.fsync
+          end
+        end
+      end
+    end
+  end
+end