logstash-output-qingstor 0.1.3 → 0.2.0

data/lib/logstash/outputs/qingstor/uploader.rb

@@ -1,64 +1,69 @@
 # encoding: utf-8
-require "qingstor/sdk"
-require "concurrent"
-require "digest/md5"
-require "base64"
+
+require 'logstash/outputs/qingstor'
+require 'qingstor/sdk'
+require 'concurrent'
+require 'digest/md5'
+require 'base64'
+
 module LogStash
   module Outputs
     class Qingstor
-      class Uploader 
+      class Uploader
         TIME_BEFORE_RETRYING_SECONDS = 1
-        DEFAULT_THREADPOOL = Concurrent::ThreadPoolExecutor.new({ :min_thread => 1,
-                                                                  :max_thread => 8,
-                                                                  :max_queue => 1,
-                                                                  :fallback_policy => :caller_runs
-                                                                })
-        attr_reader :bucket, :upload_options, :logger 
+        DEFAULT_THREADPOOL = Concurrent::ThreadPoolExecutor.new(
+          :min_thread => 1,
+          :max_thread => 8,
+          :max_queue => 1,
+          :fallback_policy => :caller_runs
+        )
+
+        attr_reader :bucket, :upload_options, :logger
 
         def initialize(bucket, logger, threadpool = DEFAULT_THREADPOOL)
           @bucket = bucket
           @logger = logger
           @workers_pool = threadpool
-        end 
+        end
 
         def upload_async(file, options = {})
           @workers_pool.post do
             upload(file, options)
-          end 
-        end 
+          end
+        end
 
         def upload(file, options = {})
           upload_options = options.fetch(:upload_options, {})
 
           file_md5 = Digest::MD5.file(file.path).to_s
-    
+
           upload_headers = {
-            "content_md5" => file_md5,
-            "body" => ::File.open(file.path)
+            'content_md5' => file_md5,
+            'body' => ::File.open(file.path)
           }
 
-          if !upload_options[:server_side_encryption_algorithm].nil?
-            base64_key = Base64.strict_encode64 upload_options[:customer_key]
-            key_md5 = Digest::MD5.hexdigest upload_options[:customer_key]
-            base64_key_md5 = Base64.strict_encode64 key_md5
-            upload_headers.merge!({
-              "x_qs_encryption_customer_algorithm" => upload_options[:server_side_encryption_algorithm],
-              "x_qs_encryption_customer_key" => base64_key,
-              "x_qs_encryption_customer_key_md5" => base64_key_md5,
-            })
-          end 
-          @logger.debug("uploading file", :file => file.key)
+          unless upload_options[:server_side_encryption_algorithm].nil?
+            base64_key = Base64.strict_encode64(upload_options[:customer_key])
+            key_md5 = Digest::MD5.hexdigest(upload_options[:customer_key])
+            base64_key_md5 = Base64.strict_encode64(key_md5)
+            upload_headers.merge!(
+              'x_qs_encryption_customer_algorithm' =>
+                upload_options[:server_side_encryption_algorithm],
+              'x_qs_encryption_customer_key' => base64_key,
+              'x_qs_encryption_customer_key_md5' => base64_key_md5
+            )
+          end
+          @logger.debug('uploading file', :file => file.key)
           bucket.put_object(file.key, upload_headers)
 
           options[:on_complete].call(file) unless options[:on_complete].nil?
-        end 
+        end
 
-        def stop 
+        def stop
           @workers_pool.shutdown
           @workers_pool.wait_for_termination(nil)
-        end 
-      end 
-    end 
-  end 
-end 
-          
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/qingstor.rb

@@ -1,37 +1,41 @@
 # encoding: utf-8
-require "logstash-core"
-require "logstash/outputs/base"
-require "logstash/namespace"
-require "tmpdir"
-require "qingstor/sdk"
-require "concurrent"
+
+require 'logstash-core'
+require 'logstash/outputs/base'
+require 'logstash/namespace'
+require 'tmpdir'
+require 'qingstor/sdk'
+require 'concurrent'
 
 class LogStash::Outputs::Qingstor < LogStash::Outputs::Base
-  require "logstash/outputs/qingstor/temporary_file"
-  require "logstash/outputs/qingstor/temporary_file_factory"
-  require "logstash/outputs/qingstor/file_repository"
-  require "logstash/outputs/qingstor/rotation_policy"
-  require "logstash/outputs/qingstor/uploader"
-  require "logstash/outputs/qingstor/qingstor_validator"
+  require 'logstash/outputs/qingstor/temporary_file'
+  require 'logstash/outputs/qingstor/temporary_file_factory'
+  require 'logstash/outputs/qingstor/file_repository'
+  require 'logstash/outputs/qingstor/rotation_policy'
+  require 'logstash/outputs/qingstor/uploader'
+  require 'logstash/outputs/qingstor/qingstor_validator'
 
   PERIODIC_CHECK_INTERVAL_IN_SECONDS = 15
-  CRASH_RECOVERY_THREADPOOL = Concurrent::ThreadPoolExecutor.new({
-                                                                  :min_threads => 1,
-                                                                  :max_threads => 2,
-                                                                  :fallback_policy => :caller_runs
-                                                                })
-  config_name "qingstor"
-
-  # When configured as :single a single instance of the Output will be shared among the
-  # pipeline worker threads. Access to the `#multi_receive/#multi_receive_encoded/#receive` method will be synchronized
-  # i.e. only one thread will be active at a time making threadsafety much simpler.
+  CRASH_RECOVERY_THREADPOOL = Concurrent::ThreadPoolExecutor.new(
+    :min_threads => 1,
+    :max_threads => 2,
+    :fallback_policy => :caller_runs
+  )
+
+  config_name 'qingstor'
+
+  # When configured as :single a single instance of the Output will be shared
+  # among the pipeline worker threads. Access to the `#multi_receive/
+  # #multi_receive_encoded/#receive` method will be synchronized i.e. only one
+  # thread will be active at a time making threadsafety much simpler.
   #
-  # You can set this to :shared if your output is threadsafe. This will maximize
-  # concurrency but you will need to make appropriate uses of mutexes in `#multi_receive/#receive`.
+  # You can set this to :shared if your output is threadsafe. This will
+  # maximize concurrency but you will need to make appropriate uses of mutexes
+  # in `#multi_receive/#receive`.
   #
-  # Only the `#multi_receive/#multi_receive_encoded` methods need to actually be threadsafe, the other methods
-  # will only be executed in a single thread
-  concurrency :shared
+  # Only the `#multi_receive/#multi_receive_encoded` methods need to actually
+  # be threadsafe, the other methods will only be executed in a single thread
+  concurrency :single
 
   # The key id to access your QingStor
   config :access_key_id, :validate => :string, :required => true
@@ -49,76 +53,85 @@ class LogStash::Outputs::Qingstor < LogStash::Outputs::Base
   config :bucket, :validate => :string, :required => true
 
   # The region of the QingStor bucket
-  config :region, :validate => ["pek3a", "sh1a"], :default => "pek3a"
+  config :region, :validate => %w[pek3a sh1a], :default => 'pek3a'
 
-  # The prefix of filenames which will work as directory in qingstor 
+  # The prefix of filenames which will work as directory in qingstor
   config :prefix, :validate => :string, :default => ''
 
-  # Set the directory where logstash store the tmp files before 
-  # sending it to qingstor, default directory in linux /tmp/logstash2qingstor
-  config :tmpdir, :validate => :string, :default => File.join(Dir.tmpdir, "logstash2qingstor")
+  # Set the directory where logstash store the tmp files before
+  # sending it to QingStor, default directory in linux /tmp/logstash2qingstor
+  config :tmpdir, :validate => :string,
+                  :default => File.join(Dir.tmpdir, 'logstash2qingstor')
 
-  # Define tags to append to the file on the qingstor bucket
+  # Define tags to append to the file on the QingStor bucket
   config :tags, :validate => :array, :default => []
 
   # Specify the content encoding. Supports ("gzip"), defaults to "none"
-  config :encoding, :validate => ["gzip", "none"], :default => "none"
+  config :encoding, :validate => %w[gzip none], :default => 'none'
 
-  # Define the strategy to use to decide when we need to rotate the file and push it to S3,
-  # The default strategy is to check for both size and time, the first one to match will rotate the file.
-  config :rotation_strategy, :validate => ["size_and_time", "size", "time"], :default => "size_and_time"
+  # Define the strategy to use to decide when we need to rotate the file
+  # and push it to QingStor.
+  # The default strategy is to check for both size and time, the first one to
+  # match will rotate the file.
+  config :rotation_strategy, :validate => %w[size_and_time size time],
+                             :default => 'size_and_time'
 
-  # Define the size requirement for each file to upload to qingstor. In byte.
+  # Define the size requirement for each file to upload to QingStor. In byte.
   config :file_size, :validate => :number, :default => 1024 * 1024 * 5
 
-  # Define the time interval for each file to upload to qingstor. In minutes.
-  config :file_time, :validate => :number, :default => 15 
+  # Define the time interval for each file to upload to QingStor. In minutes.
+  config :file_time, :validate => :number, :default => 15
 
-  # Specify maximum number of workers to use to upload the files to Qingstor
-  config :upload_workers_count, :validate => :number, :default => (Concurrent.processor_count * 0.5).ceil
+  # Specify maximum number of workers to use to upload the files to QingStor
+  config :upload_workers_count,
+         :validate => :number,
+         :default => (Concurrent.processor_count * 0.5).ceil
 
   # Number of items we can keep in the local queue before uploading them
-  config :upload_queue_size, :validate => :number, :default => 2 * (Concurrent.processor_count * 0.25).ceil
+  config :upload_queue_size,
+         :validate => :number,
+         :default => 2 * (Concurrent.processor_count * 0.25).ceil
 
   # Specifies what type of encryption to use when SSE is enabled.
-  config :server_side_encryption_algorithm, :validate => ["AES256", "none"], :default => "none"
+  config :server_side_encryption_algorithm, :validate => %w[AES256 none],
+                                            :default => 'none'
 
-  # Specifies the encryption customer key that would be used in server side   
+  # Specifies the encryption customer key that would be used in server side
   config :customer_key, :validate => :string
 
-  # Specifies if set to true, it would upload existing file in targeting folder at the beginning.
-  config :restore, :validate => :boolean, :default => false  
+  # Specifies if set to true, it would upload existing file in targeting folder
+  # when the plugin is launched
+  config :restore, :validate => :boolean, :default => false
 
-  public
   def register
     QingstorValidator.prefix_valid?(@prefix) unless @prefix.nil?
 
-    if !directory_valid?(@tmpdir)
-      raise LogStash::ConfigurationError, "Logstash must have the permissions to write to the temporary directory: #{@tmpdir}"
+    unless directory_valid?(@tmpdir)
+      raise LogStash::ConfigurationError,
+            "Logstash must have the permissions to write to: #{@tmpdir}"
     end
-    
+
     @file_repository = FileRepository.new(@tags, @encoding, @tmpdir)
 
     @rotation = RotationPolicy.new(@rotation_strategy, @file_size, @file_time)
 
-    executor = Concurrent::ThreadPoolExecutor.new({ 
+    executor = Concurrent::ThreadPoolExecutor.new(
       :min_threads => 1,
       :max_threads => @upload_workers_count,
       :max_queue => @upload_queue_size,
-      :fallback_policy => :caller_runs 
-    })
+      :fallback_policy => :caller_runs
+    )
 
-    @qs_bucket = get_bucket
+    @qs_bucket = getbucket
     QingstorValidator.bucket_valid?(@qs_bucket)
 
     @uploader = Uploader.new(@qs_bucket, @logger, executor)
 
     start_periodic_check if @rotation.needs_periodic?
 
-    restore_from_crash if @restore 
+    restore_from_crash if @restore
   end # def register
 
-  public
   def multi_receive_encoded(events_and_encoded)
     prefix_written_to = Set.new
 
@@ -127,18 +140,21 @@ class LogStash::Outputs::Qingstor < LogStash::Outputs::Base
       prefix_written_to << prefix_key
 
       begin
-        @file_repository.get_file(prefix_key) { |file| file.write(encoded + "\n") }
+        @file_repository.get_file(prefix_key) do |f|
+          content = encoded.strip + "\n"
+          f.write(content)
+        end
       rescue Errno::ENOSPC => e
-        @logger.error("QingStor: Nospace left in temporary directory", :tmpdir => @tmpdir)
-        raise e 
-      end 
-    end # end of each method  
-
-    # check the file after file writing 
-    # Groups IO calls to optimize fstat checks 
+        @logger.error('QingStor: Nospace left in temporary directory',
+                      :tmpdir => @tmpdir)
+        raise e
+      end
+    end # end of each method
+
+    # check the file after file writing
+    # Groups IO calls to optimize fstat checks
     rotate_if_needed(prefix_written_to)
-  end  # def multi_receive_encoded
-  
+  end # def multi_receive_encoded
 
   def rotate_if_needed(prefixs)
     prefixs.each do |prefix|
@@ -146,103 +162,107 @@ class LogStash::Outputs::Qingstor < LogStash::Outputs::Base
         tmp_file = factory.current
 
         if @rotation.rotate?(tmp_file)
-          @logger.debug("Rotate file", 
+          @logger.debug('Rotate file',
                         :strategy => tmp_file.key,
                         :path => tmp_file.path)
           upload_file(tmp_file)
           factory.rotate!
-        end 
-      end 
-    end 
+        end
+      end
+    end
   end # def rotate_if_needed
 
   def upload_file(file)
-    @logger.debug("Add file to uploading queue", :key => file.key)
-    file.close 
-    @logger.debug("upload options", :upload_options => upload_options)
+    @logger.debug('Add file to uploading queue', :key => file.key)
+    file.close
+    @logger.debug('upload options', :upload_options => upload_options)
     @uploader.upload_async(file,
-                          :on_complete => method(:clean_temporary_file),
-                          :upload_options => upload_options)
-  end 
+                           :on_complete => method(:clean_temporary_file),
+                           :upload_options => upload_options)
+  end
 
-  def get_bucket
+  def getbucket
     @qs_config = QingStor::SDK::Config.init @access_key_id, @secret_access_key
-    @qs_config.update({ host: @host, port: @port }) unless @host.nil?
+    @qs_config.update(:host => @host, :port => @port) unless @host.nil?
     @qs_service = QingStor::SDK::Service.new @qs_config
     @qs_service.bucket @bucket, @region
-  end 
+  end
 
-  def close 
+  def close
     stop_periodic_check if @rotation.needs_periodic?
 
-    @logger.debug("uploading current workspace")
+    @logger.debug('uploading current workspace')
     @file_repository.each_files do |file|
       upload_file(file)
-    end 
+    end
 
     @file_repository.shutdown
 
-    @uploader.stop 
+    @uploader.stop
 
     @crash_uploader.stop if @restore
-  end 
+  end
 
   def upload_options
     options = {
-      :content_encoding => @encoding == "gzip" ? "gzip" : nil 
+      :content_encoding => @encoding == 'gzip' ? 'gzip' : nil
     }
 
-    if  @server_side_encryption_algorithm == "AES256" && !@customer_key.nil?
-      options.merge!({
-        :server_side_encryption_algorithm => @server_side_encryption_algorithm,
-        :customer_key => @customer_key
-      })
-    end 
-    
-    options 
-  end 
+    if @server_side_encryption_algorithm == 'AES256' && !@customer_key.nil?
+      options[:server_side_encryption_algorithm] = @server_side_encryption_algorithm
+      options[:customer_key] = @customer_key
+    end
+
+    options
+  end
 
   def clean_temporary_file(file)
-    @logger.debug("Removing temporary file", :file => file.path)
+    @logger.debug('Removing temporary file', :file => file.path)
     file.delete!
-  end 
+  end
 
   def start_periodic_check
-    @logger.debug("Start periodic rotation check")
+    @logger.debug('Start periodic rotation check')
 
-    @periodic_check = Concurrent::TimerTask.new(:execution_interval => PERIODIC_CHECK_INTERVAL_IN_SECONDS) do
-      @logger.debug("Periodic check for stale files")
+    @periodic_check = Concurrent::TimerTask.new(
+      :execution_interval => PERIODIC_CHECK_INTERVAL_IN_SECONDS
+    ) do
+      @logger.debug('Periodic check for stale files')
 
       rotate_if_needed(@file_repository.keys)
     end
 
-    @periodic_check.execute 
-  end 
+    @periodic_check.execute
+  end
 
-  def stop_periodic_check 
-    @periodic_check.shutdown 
-  end 
+  def stop_periodic_check
+    @periodic_check.shutdown
+  end
 
   def directory_valid?(path)
-    begin 
-      FileUtils.mkdir_p(path) unless Dir.exist?(path)
-      ::File.writable?(path)
-    rescue 
-      false 
-    end 
-  end 
+    FileUtils.mkdir_p(path) unless Dir.exist?(path)
+    ::File.writable?(path)
+  rescue
+    false
+  end
 
-  def restore_from_crash 
-    @crash_uploader = Uploader.new(@qs_bucket, @logger, CRASH_RECOVERY_THREADPOOL)
+  def restore_from_crash
+    @crash_uploader = Uploader.new(@qs_bucket, @logger,
+                                   CRASH_RECOVERY_THREADPOOL)
 
     temp_folder_path = Pathname.new(@tmpdir)
-    Dir.glob(::File.join(@tmpdir, "**/*"))
-      .select { |file| ::File.file?(file) }
-      .each do |file| 
-      temp_file = TemporaryFile.create_from_existing_file(file, temp_folder_path)
-      @logger.debug("Recoving from crash and uploading", :file => temp_file.path)
-      @crash_uploader.upload_async(temp_file, :on_complete => method(:clean_temporary_file),
-                                              :upload_options => upload_options)
-    end 
-  end 
-end # class LogStash::Outputs::Qingstor
+    Dir.glob(::File.join(@tmpdir, '**/*'))
+       .select { |file| ::File.file?(file) }
+       .each do |file|
+      temp_file = TemporaryFile.create_from_existing_file(file,
+                                                          temp_folder_path)
+      @logger.debug('Recoving from crash and uploading',
+                    :file => temp_file.path)
+      @crash_uploader.upload_async(
+        temp_file,
+        :on_complete => method(:clean_temporary_file),
+        :upload_options => upload_options
+      )
+    end
+  end
+end # class LogStash::Outputs::Qingstor

data/logstash-output-qingstor.gemspec

@@ -1,28 +1,28 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-qingstor'
-  s.version       = '0.1.3'
+  s.version       = '0.2.0'
   s.licenses      = ['Apache License (2.0)']
   s.summary       = 'logstash output plugin for qingstor'
   s.description   = 'Collect the outputs of logstash and store into Qingstor'
-  s.homepage      = 'https://github.com/tacinight/logstash-output-qingstor'
+  s.homepage      = 'https://github.com/yunify/logstash-output-qingstor'
   s.authors       = ['Evan Zhao']
   s.email         = 'tacingiht@gmail.com'
   s.require_paths = ['lib']
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
-   # Tests
+  s.files = Dir['lib/**/*', 'spec/**/*', 'vendor/**/*', '*.gemspec', '*.md', 'CONTRIBUTORS', 'Gemfile', 'LICENSE', 'NOTICE.TXT']
+  # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
 
   # Special flag to let us know this is actually a logstash plugin
-  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+  s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'output' }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
-  s.add_runtime_dependency "logstash-codec-plain"
-  s.add_runtime_dependency "qingstor-sdk", ">=1.9.2"
-  s.add_runtime_dependency "concurrent-ruby"
-  
-  s.add_development_dependency "stud", "~> 0.0.22"
-  s.add_development_dependency "logstash-devutils"
+  s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0'
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'qingstor-sdk', '>=1.9.2'
+  s.add_runtime_dependency 'concurrent-ruby'
+
+  s.add_development_dependency 'stud', '~> 0.0.22'
+  s.add_development_dependency 'logstash-devutils'
 end

data/spec/outputs/qingstor/file_repository_spec.rb

@@ -1,26 +1,28 @@
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/outputs/qingstor/temporary_file"
-require "logstash/outputs/qingstor/temporary_file_factory"
-require "logstash/outputs/qingstor/file_repository"
-require "tmpdir"
-
-describe LogStash::Outputs::Qingstor::FileRepository do 
-  let(:tags) { ["tag1", "tag2", "tag3"]}
-  let(:encoding) { "none" }
-  let(:tmpdir) { File.join(Dir.tmpdir, "lg2qs") }
-  let(:prefix) { "aprefix" }
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/qingstor/temporary_file'
+require 'logstash/outputs/qingstor/temporary_file_factory'
+require 'logstash/outputs/qingstor/file_repository'
+require 'tmpdir'
 
+describe LogStash::Outputs::Qingstor::FileRepository do
   subject { described_class.new(tags, encoding, tmpdir) }
 
-  it "can get current file io" do 
-    subject.get_file(prefix) do |file| 
-       expect(file).to be_kind_of(LogStash::Outputs::Qingstor::TemporaryFile)
-    end 
-  end 
+  let(:tags) { %w[tag1 tag2 tag3] }
+  let(:encoding) { 'none' }
+  let(:tmpdir) { File.join(Dir.tmpdir, 'lg2qs') }
+  let(:prefix) { 'aprefix' }
+
+  it 'can get current file io' do
+    subject.get_file(prefix) do |file|
+      expect(file).to be_kind_of(LogStash::Outputs::Qingstor::TemporaryFile)
+    end
+  end
 
-  it "can get current file factory" do 
-    subject.get_factory(prefix) do |factory| 
-      expect(factory).to be_kind_of(LogStash::Outputs::Qingstor::TemporaryFileFactory)
-    end 
-  end 
-end 
+  it 'can get current file factory' do
+    subject.get_factory(prefix) do |factory|
+      expect(factory).to be_kind_of(
+        LogStash::Outputs::Qingstor::TemporaryFileFactory
+      )
+    end
+  end
+end

data/spec/outputs/qingstor/qingstor_validator_spec.rb

@@ -1,21 +1,23 @@
 # encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/outputs/qingstor/qingstor_validator"
-require_relative "../qs_access_helper"
+
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/qingstor/qingstor_validator'
+require_relative '../qs_access_helper'
 
 describe LogStash::Outputs::Qingstor::QingstorValidator do
-  let(:normal_prefix) { "super/bucket" }
-  let(:wrong_prefix1) { "/wrong/prefix" }
+  let(:normal_prefix) { 'super/bucket' }
+  let(:wrong_prefix1) { '/wrong/prefix' }
   let(:wrong_prefix2) { normal_prefix * 100 }
   let(:bucket) { qs_bucket_init }
 
-  it "raise error if the prefix is not valid" do 
-    expect{ described_class.prefix_valid?(wrong_prefix1) }.to raise_error(LogStash::ConfigurationError)
-    expect{ described_class.prefix_valid?(wrong_prefix2) }.to raise_error(LogStash::ConfigurationError)
+  it 'raise error if the prefix is not valid' do
+    expect { described_class.prefix_valid?(wrong_prefix1) }
+      .to raise_error(LogStash::ConfigurationError)
+    expect { described_class.prefix_valid?(wrong_prefix2) }
+      .to raise_error(LogStash::ConfigurationError)
   end
 
-  it "return true if the prefix is valid" do 
+  it 'return true if the prefix is valid' do
     expect(described_class.prefix_valid?(normal_prefix)).to be_truthy
-  end 
-
-end 
+  end
+end