logstash-output-qingstor 0.3.3 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 515c9929fb2a985fa7e97b8775615a2a57fee0c7ba8d7a1cafeb89d309fa55cf
-  data.tar.gz: 81e0dea2bd38b84f353f51505699ced9630ead77318654621a749271bbc5ed15
+  metadata.gz: 3c17d2f3c3498c58c9a99bfb78fb39e1dec09d1dd72d8a4c8c56c0c1ea500855
+  data.tar.gz: dc01d1554f162363e75876155f0338b1efb740804cd673fd8afc047f51f02b71
 SHA512:
-  metadata.gz: 9f5b3b2496db39fc228404ca1b6f25d7e0f4f55d4499f7276c26795be7e02c3f798d7654cf1a75759b324de3eba7d596fbfcb5df739fcd34a0c31bb666e74a51
-  data.tar.gz: 6157494e770ff5c82a4b803af6c840087b7612d4c3d54461788b115203495513156476fbc0d6493ebb1475065e4ab63db13d01157b6011213b3d19a40f09cbfc
+  metadata.gz: 0c23b81601ac731bccb9214d45b67f9b44917947afa7de30e264972ace16720c4b1e8d309beefda4b97f8f1d5cdbe550d5504e3d19fdb1c8ec6adc53979792dc
+  data.tar.gz: 5c06847d78b3f526b0ca671561923964a28a55f096f7dd17d73c66a2d56afa53b433eedcf93373a3e1a7a47c5006a89a3b69aa472e01381f16145b2c0853923e

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.3.4
+  - Change some events log level to 'info'
+
 ## 0.3.2
   - Fix the errors of unit conversion in rotation policy
 

data/lib/logstash/outputs/qingstor/multipart_uploader.rb

@@ -64,7 +64,7 @@ module LogStash
             object_parts = (0..last_part_number).to_a.map {|x| {'part_number' => x}}
             @upload_headers['object_parts'] = object_parts
             res = @bucket.complete_multipart_upload(@object.key, @upload_headers)
-            @logger.debug('multipart uploading completed', :file => @object.key)
+            @logger.info('multipart uploading completed', :file => @object.key)
           end 
         
           def calculate_segment

data/lib/logstash/outputs/qingstor/rotation_policy.rb

@@ -7,10 +7,6 @@ module LogStash
     class Qingstor
       class RotationPolicy
         class Policy
-          def to_s
-            name
-          end
-
           def name
             self.class.name.split('::').last.downcase
           end
@@ -36,6 +32,10 @@ module LogStash
           def rotate?(file)
             !file.empty? && (::Time.now - file.ctime) >= @file_time
           end
+
+          def to_s
+            { :policy => name, :file_time => @file_time }.to_s
+          end
         end
 
         class Size < Policy
@@ -49,6 +49,10 @@ module LogStash
           end
 
           def needs_periodic?; false; end
+
+          def to_s
+            { :policy => name, :file_size => @file_size }.to_s
+          end
         end
 
         class SizeAndTime < Policy
@@ -61,6 +65,11 @@ module LogStash
             (!file.empty? && (::Time.now - file.ctime) >= @file_time) ||
               (file.size >= @file_size)
           end
+
+          def to_s
+            { :policy => name, :file_time => @file_time,
+              :file_size => @file_size }.to_s
+          end
         end
 
         def Policy(policy, file_size, file_time)

data/lib/logstash/outputs/qingstor/uploader.rb

@@ -39,13 +39,13 @@ module LogStash
           upload_headers = process_encrypt_options(upload_options)
 
           if file.size > 50 * 1024 * 1024
-            @logger.debug('multipart uploading file', :file => file.key)
+            @logger.info('Multipart uploading file', :file => file.key)
             multipart_uploader = MultipartUploader.new(@bucket, @logger, file, upload_headers)
             multipart_uploader.upload
           else
             upload_headers['content_md5'] = Digest::MD5.file(file.path).to_s
             upload_headers['body'] = ::File.read(file.path)
-            @logger.debug('uploading file', :file => file.key)
+            @logger.info('Uploading file', :file => file.key)
             @bucket.put_object(file.key, upload_headers)
           end
 

data/lib/logstash/outputs/qingstor.rb

@@ -105,16 +105,13 @@ class LogStash::Outputs::Qingstor < LogStash::Outputs::Base
 
   def register
     QingstorValidator.prefix_valid?(@prefix) unless @prefix.nil?
-
     unless directory_valid?(@tmpdir)
       raise LogStash::ConfigurationError,
             "Logstash must have the permissions to write to: #{@tmpdir}"
     end
 
     @file_repository = FileRepository.new(@tags, @encoding, @tmpdir)
-
     @rotation = RotationPolicy.new(@rotation_strategy, @file_size, @file_time)
-
     executor = Concurrent::ThreadPoolExecutor.new(
       :min_threads => 1,
       :max_threads => @upload_workers_count,
@@ -124,14 +121,22 @@ class LogStash::Outputs::Qingstor < LogStash::Outputs::Base
 
     @qs_bucket = getbucket
     QingstorValidator.bucket_valid?(@qs_bucket)
-
     @uploader = Uploader.new(@qs_bucket, @logger, executor)
 
+    log_print_config
     start_periodic_check if @rotation.needs_periodic?
-
     restore_from_crash if @restore
   end # def register
 
+  def log_print_config
+    @logger.info('Run at setting: ', :prefix => @prefix,
+                                     :tmpdir => @tmpdir,
+                                     :rotation => @rotation.to_s,
+                                     :tags => @tags,
+                                     :encoding => @encoding,
+                                     :restore => @restore)
+  end
+
   def multi_receive_encoded(events_and_encoded)
     prefix_written_to = Set.new
 
@@ -191,7 +196,7 @@ class LogStash::Outputs::Qingstor < LogStash::Outputs::Base
   def close
     stop_periodic_check if @rotation.needs_periodic?
 
-    @logger.debug('uploading current workspace')
+    @logger.info('uploading current workspace before closing')
     @file_repository.each_files do |file|
       upload_file(file) if file.size > 0
     end
@@ -217,12 +222,12 @@ class LogStash::Outputs::Qingstor < LogStash::Outputs::Base
   end
 
   def clean_temporary_file(file)
-    @logger.debug('Removing temporary file', :file => file.path)
+    @logger.info('Callback: removing temporary file', :file => file.path)
     file.delete!
   end
 
   def start_periodic_check
-    @logger.debug('Start periodic rotation check')
+    @logger.info('Start periodic rotation check')
 
     @periodic_check = Concurrent::TimerTask.new(
       :execution_interval => PERIODIC_CHECK_INTERVAL_IN_SECONDS
@@ -259,7 +264,7 @@ class LogStash::Outputs::Qingstor < LogStash::Outputs::Base
       # Now multipart uploader supports file size up to 500GB
       if temp_file.size > 0
         temp_file.key = 'Restored/' + Time.new.strftime('%Y-%m-%d/') + temp_file.key
-        @logger.debug('Recoving from crash and uploading',
+        @logger.info('Recoving from crash and uploading',
                     :file => temp_file.key)
         @crash_uploader.upload_async(
           temp_file,
@@ -267,7 +272,7 @@ class LogStash::Outputs::Qingstor < LogStash::Outputs::Base
           :upload_options => upload_options
         )
       elsif temp_file.size == 0
-        @logger.debug('Recoving from crash, delete empty files',
+        @logger.info('Recoving from crash, delete empty files',
                     :file => temp_file.path)
         temp_file.delete!
       end

data/logstash-output-qingstor.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-qingstor'
-  s.version       = '0.3.3'
+  s.version       = '0.3.4'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'logstash output plugin for qingstor'
   s.description   = 'Collect the outputs of logstash and store into QingStor'

data/spec/outputs/qingstor/rotation_policy_spec.rb

@@ -5,18 +5,18 @@ require 'logstash/outputs/qingstor/temporary_file'
 require 'logstash/outputs/qingstor/rotation_policy'
 
 describe LogStash::Outputs::Qingstor::RotationPolicy do
-  let(:size_file) { 5 }
-  let(:time_file) { 2 }
+  let(:file_size) { 5 }
+  let(:file_time) { 2 }
 
   shared_examples 'size rotation' do
-    it 'raise error if size_file is no grater then 0' do
+    it 'raise error if file_size is no grater then 0' do
       expect { described_class.new('size', 0, 0) }
         .to raise_error(LogStash::ConfigurationError)
       expect { described_class.new('size', -1, 0) }
         .to raise_error(LogStash::ConfigurationError)
     end
 
-    it "return true if the file has a bigger size value then 'size_file'" do
+    it "return true if the file has a bigger size value then 'file_size'" do
       file = double('file')
       allow(file).to receive(:ctime) { Time.now }
       allow(file).to receive(:empty?) { true }
@@ -34,7 +34,7 @@ describe LogStash::Outputs::Qingstor::RotationPolicy do
   end
 
   shared_examples 'time rotation' do
-    it 'raise error if time_file is no grater then 0' do
+    it 'raise error if file_time is no grater then 0' do
       expect { described_class.new('time', 0, 0) }
         .to raise_error(LogStash::ConfigurationError)
       expect { described_class.new('time', 0, -1) }
@@ -51,7 +51,7 @@ describe LogStash::Outputs::Qingstor::RotationPolicy do
 
     it 'return false if the file is old enough with file size 0' do
       file = double('file')
-      allow(file).to receive(:ctime) { Time.now - (time_file * 2 * 60) }
+      allow(file).to receive(:ctime) { Time.now - (file_time * 2 * 60) }
       allow(file).to receive(:empty?) { true }
       allow(file).to receive(:size) { 0 }
       expect(subject.rotate?(file)).to be_falsey
@@ -59,7 +59,7 @@ describe LogStash::Outputs::Qingstor::RotationPolicy do
 
     it 'return truth if the file is old enough and non-empty' do
       file = double('file')
-      allow(file).to receive(:ctime) { Time.now - (time_file * 2 * 60) }
+      allow(file).to receive(:ctime) { Time.now - (file_time * 2 * 60) }
       allow(file).to receive(:empty?) { false }
       allow(file).to receive(:size) { 5 * 1024 * 1024 }
       expect(subject.rotate?(file)).to be_truthy
@@ -67,21 +67,41 @@ describe LogStash::Outputs::Qingstor::RotationPolicy do
   end
 
   context 'when time policy' do
-    subject { described_class.new('time', size_file, time_file) }
+    subject { described_class.new('time', file_size, file_time) }
 
     include_examples 'time rotation'
+
+    it 'get suitable description of the class' do
+      time = file_time * 60
+      expect(subject.to_s).to eq({:policy=>"time", 
+                                  :file_time=>time}.to_s)
+    end
   end
 
   context 'when size policy' do
-    subject { described_class.new('size', size_file, time_file) }
+    subject { described_class.new('size', file_size, file_time) }
 
     include_examples 'size rotation'
+
+    it 'get suitable description of the class' do
+      size = file_size * 1024 * 1024
+      expect(subject.to_s).to eq({:policy=>"size", 
+                                  :file_size=>size}.to_s)
+    end
   end
 
   context 'when size_and_time policy' do
-    subject { described_class.new('size_and_time', size_file, time_file) }
+    subject { described_class.new('size_and_time', file_size, file_time) }
 
     include_examples 'time rotation'
     include_examples 'size rotation'
+
+    it 'get suitable description of the class' do
+      size = file_size * 1024 * 1024
+      time = file_time * 60
+      expect(subject.to_s).to eq({:policy=>"sizeandtime", 
+                                  :file_time=>time,
+                                  :file_size=>size}.to_s)
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-qingstor
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - Evan Zhao
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-01-18 00:00:00.000000000 Z
+date: 2018-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement