logstash-output-qingstor 0.2.6 → 0.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6fe51f69908599f882c6ac6b6a9afc08e0770b8bffda5679ab83ac5e5acf94dc
-  data.tar.gz: 0d499a3d20bf83f0a39d22663077cf13a92d0bfb8281d8aca848914665c07a21
+  metadata.gz: 46170bdbbd13271afd875490c56b4d77dc2bed5dac63a63030b69dff5895f78d
+  data.tar.gz: 8b7d9bd379b57980e61fd2b20475bc12181d7154c701de0d4f5af224e2defc47
 SHA512:
-  metadata.gz: 9c1959265a7e30b3777e09135c69013f1e70332e8772893d1d142f7fc7f0a4fb03a1f5f6409b0585766129c39617bb544f79997869dea7f9a77ec83c3e082f4d
-  data.tar.gz: '058d9cb15d999c055fe5cb8b1333dc59064861923b4f15a865fc43ceaf7f4fb6f798ceb8f3956fb6f07e46801da8eade2b2fd1c67688d7ca0ab4156af33244fb'
+  metadata.gz: 171d450378ab1852dddb0815e01d9d4f0c964a8fd81c5974cb17bfbd152fbe9fe7ebcb439c9491f598c7cfa9eb4e2ff329cd5404932e7ae3320ca070775635e1
+  data.tar.gz: b8a50149df4fd8e6e3b526eb99188559a4cca0c3f1c96a65bbb1578b2b4c3d5f11d2c108edee81f82e557f4d98c385fa7d74109eb105e018bc983ab79195ea40

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.2.7
+  - Fix: filter empty file and large file while restoring
+
 ## 0.2.6
   - Fix: swap uuid and date of the file name for sorting
   

data/lib/logstash/outputs/qingstor.rb

@@ -256,13 +256,21 @@ class LogStash::Outputs::Qingstor < LogStash::Outputs::Base
        .each do |file|
       temp_file = TemporaryFile.create_from_existing_file(file,
                                                           temp_folder_path)
-      @logger.debug('Recoving from crash and uploading',
+
+      # Restoring too large file would cause java heap out of memory
+      if temp_file.size > 0 && temp_file.size < 100 * 1024 * 1024
+        @logger.debug('Recoving from crash and uploading',
+                    :file => temp_file.path)
+        @crash_uploader.upload_async(
+          temp_file,
+          :on_complete => method(:clean_temporary_file),
+          :upload_options => upload_options
+        )
+      elsif temp_file.size == 0
+        @logger.debug('Recoving from crash, delete empty files',
                     :file => temp_file.path)
-      @crash_uploader.upload_async(
-        temp_file,
-        :on_complete => method(:clean_temporary_file),
-        :upload_options => upload_options
-      )
+        temp_file.delete!
+      end
     end
   end
 end # class LogStash::Outputs::Qingstor

data/logstash-output-qingstor.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-qingstor'
-  s.version       = '0.2.6'
+  s.version       = '0.2.7'
   s.licenses      = ['Apache License (2.0)']
   s.summary       = 'logstash output plugin for qingstor'
   s.description   = 'Collect the outputs of logstash and store into QingStor'

data/spec/outputs/qingstor_spec.rb

@@ -44,12 +44,11 @@ describe LogStash::Outputs::Qingstor do
   end
 
   it 'upload existing file if turn on restore function' do
-    unless File.exist?(File.join(tmpdir, 'uuid_dir'))
-      FileUtils.mkdir_p(File.join(tmpdir, 'uuid_dir'))
-    end
-    f = File.open(File.join(tmpdir, 'uuid_dir/temp'), 'w')
-    f.write(event_encoded * 10)
-    f.close
+    non_empty_file = File.open(File.join(tmpdir, 'non-empty-file'), 'w')
+    non_empty_file.write(event_encoded * 10)
+    non_empty_file.close
+    empty_file = File.open(File.join(tmpdir, 'empty-file'), 'w')
+    empty_file.close
     fetch_event(options.merge('restore' => true, 'tmpdir' => tmpdir),
                 events_and_encoded)
     expect(list_remote_file.size).to eq(2)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-qingstor
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.2.7
 platform: ruby
 authors:
 - Evan Zhao
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-12-01 00:00:00.000000000 Z
+date: 2017-12-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement