logstash-output-qingstor 0.2.8 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fff7cb0198f0e11e80e348703e766db542be3f787b84238a9ad130d9d33c5b28
-  data.tar.gz: 479052c30b8979f3382d2b1a30149b7a8597bbb3dc13eb073b1dadbf7647f1ab
+  metadata.gz: b634b7aee3754a9bfc4f88269a0ef6fc4135249d1143440ccae417c67852b47f
+  data.tar.gz: 05c469d151a6647bb985b15a368dc3c607d69bf6eee496e6ba1d5841185b2b75
 SHA512:
-  metadata.gz: dfb42424e15a0b6dd99e4c257ebf45276a50bcaf2efc6d5d5aeb431062568833202966bc7d7ffaafd0552ec66068bd57755516c27aacee13e39a6ddfdb38813d
-  data.tar.gz: 1978475fb39c13aa4cee4a5bb86e6d66533136b4fe116a5e513944e101ed6e9a2489ff80a0191284f789e87993853d1fdd566be12d2d945d72a621540bd70616
+  metadata.gz: 2579b496ce9f0da3c9831d4de53f70eb23c7f51febe23b7c82052df1cb2d99337c23724b34a3f8d66ddd385efe468e17f158bb8d786b4ceb5a6ee309ace88a2c
+  data.tar.gz: 545228916a49aa445d47b8242ef062c1d78834590352c7d32f81fa6cb4f93acc7208c78d3bbed7b133f5a3fbee2abd46aa6ab2dc5e7912cacb2db303dc25005e

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.3.1
+  - Refactor module: rotation policy
+  
+## 0.3.0
+  - Add MultipartUploader supporting large file size up to 500GB
+
 ## 0.2.8
   - Fix: add prefixes for restored file
 

data/lib/logstash/outputs/qingstor/multipart_uploader.rb

@@ -0,0 +1,94 @@
+# encoding: utf-8
+
+require 'logstash/outputs/qingstor'
+
+module LogStash
+  module Outputs
+    class Qingstor
+      class Uploader
+        class MultipartUploader
+          attr_reader :bucket, :logger
+          attr_accessor :upload_headers
+        
+          # According to QingStor API doc, the minimal part size is 4MB.
+          # the maximal part size is 1GB.
+          # Here the minimal size would never be used and the maximal part size is 50MB.
+          # Overlarge file size would consume java heap space and throw Exceptions.
+          MINIMAL_PART_SIZE = 4 * 1024 * 1024
+          MAXIMAL_PART_SIZE = 50 * 1024 * 1024
+          
+          def initialize(bucket, logger, object, upload_headers)
+            @bucket = bucket
+            @logger = logger
+            @object = object
+            @upload_headers = upload_headers
+          end
+        
+          def upload
+            initiate_multipart_upload
+            segment_size = calculate_segment
+            upload_multipart(segment_size)
+          end
+        
+          def initiate_multipart_upload
+            res = @bucket.initiate_multipart_upload(@object.key, @upload_headers)
+            @upload_headers['upload_id'] = res.fetch(:upload_id) do 
+              raise(QingStor::Error, :error_msg => res)
+            end
+            @logger.debug('initiated upload id', :file => @object.key, 
+                                                 :upload_id => res[:upload_id])
+            res[:upload_id]
+          end
+        
+          def upload_multipart(part_number = 0, segment_size)
+            ::File.open(@object.path) do |f|
+              f.seek(part_number * segment_size) if part_number != 0
+              while data = f.read(segment_size)
+                content_md5 = Digest::MD5.hexdigest(data)
+                @upload_headers['body'] = data
+                @upload_headers['content_length'] = segment_size
+                @upload_headers['content_md5'] = content_md5
+                @upload_headers['part_number'] = part_number
+                @logger.debug('multipart uploading: ',
+                              :file => @object.key,
+                              :part_number => part_number)
+                @bucket.upload_multipart(@object.key, @upload_headers)
+                part_number += 1
+              end
+            end
+            
+            complete_multipart_upload(part_number - 1)
+          end 
+        
+          def complete_multipart_upload(last_part_number)
+            object_parts = (0..last_part_number).to_a.map {|x| {'part_number' => x}}
+            @upload_headers['object_parts'] = object_parts
+            res = @bucket.complete_multipart_upload(@object.key, @upload_headers)
+            @logger.debug('multipart uploading completed', :file => @object.key)
+          end 
+        
+          def calculate_segment
+            segment_size = @object.size
+            
+            while segment_size >= MAXIMAL_PART_SIZE
+              segment_size /= 2.0
+              segment_size = segment_size.ceil
+            end
+        
+            segment_size
+          end
+        
+          def resume_multipart_upload(upload_id)
+            @logger.debug('resume multipart uploading', :file => @object.key,
+                                                        :upload_id => upload_id)
+            @upload_headers['upload_id'] = upload_id
+            res = @bucket.list_multipart(@object.key, upload_headers)
+            segment_size = res[:object_parts][0][:size]
+            part_number = res[:object_parts].count
+            upload_multipart(part_number, segment_size)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/qingstor/rotation_policy.rb

@@ -6,71 +6,86 @@ module LogStash
   module Outputs
     class Qingstor
       class RotationPolicy
-        def initialize(policy, file_size, file_time)
-          @policy = policy
-          case policy
-          when 'time'
-            init_time(file_time)
-          when 'size'
-            init_size(file_size)
-          when 'size_and_time'
-            init_size(file_size)
-            init_time(file_time)
+        class Policy
+          def to_s
+            name
           end
-        end
 
-        def init_size(file_size)
-          if file_size <= 0
-            raise LogStash::ConfigurationError, "'file_size' need to "\
-              + 'be greater than 0'
+          def name
+            self.class.name.split('::').last.downcase
+          end
+
+          def needs_periodic?
+            true
+          end
+
+          def positive_check(*arg)
+            arg.each do |x|
+              raise(LogStash::ConfigurationError,
+                    "#{name} policy needs positive arguments") if x <= 0
+            end
           end
-          @file_size = file_size
         end
 
-        def init_time(file_time)
-          if file_time <= 0
-            raise LogStash::ConfigurationError, "'file_time' need to "\
-              + 'be greater than 0'
+        class Time < Policy
+          def initialize(file_size, file_time)
+            @file_time = file_time
+            positive_check(@file_time)
+          end
+
+          def rotate?(file)
+            !file.empty? && (::Time.now - file.ctime) >= @file_time
           end
-          @file_time = file_time * 60
+        end 
+
+        class Size < Policy
+          def initialize(file_size, file_time)
+            @file_size = file_size
+            positive_check(@file_size)
+          end
+
+          def rotate?(file)
+            file.size >= @file_size
+          end
+
+          def needs_periodic?; false; end
         end
 
-        def rotate?(file)
-          case @policy
-          when 'time'
-            time_rotate?(file)
-          when 'size'
-            size_rotate?(file)
-          when 'size_and_time'
-            size_and_time_rotate?(file)
+        class SizeAndTime < Policy
+          def initialize(file_size, file_time)
+            @file_size, @file_time = file_size, file_time
+            positive_check(file_size, file_time)
+          end
+
+          def rotate?(file)
+            (!file.empty? && (::Time.now - file.ctime) >= @file_time) ||
+            (file.size >= @file_size)
           end
         end
 
-        def size_and_time_rotate?(file)
-          size_rotate?(file) || time_rotate?(file)
+        def Policy(policy, file_size, file_time)
+          case policy
+          when Policy then policy
+          else 
+            self.class.const_get(policy.to_s.split('_').map(&:capitalize).join)
+              .new(file_size, file_time)
+          end
         end
 
-        def size_rotate?(file)
-          file.size >= @file_size
+        def initialize(policy, file_size, file_time)
+          @policy = Policy(policy, file_size, file_time)
         end
 
-        def time_rotate?(file)
-          !file.empty? && (Time.now - file.ctime) >= @file_time
+        def rotate?(file)
+          @policy.rotate?(file)
         end
 
         def needs_periodic?
-          case @policy
-          when 'time' then
-            true
-          when 'size_and_time' then
-            true
-          else
-            false
-          end
+          @policy.needs_periodic?
         end
 
         def to_s
-          "#{@policy} #{@file_time} #{@file_size}"
+          @policy.to_s
         end
       end
     end

data/lib/logstash/outputs/qingstor/temporary_file.rb

@@ -49,7 +49,7 @@ module LogStash
           rescue
             IOError
           end
-          FileUtils.rm(::File.join(@tmp_path, @key), :force => true)
+          FileUtils.rm(path, :force => true)
         end
 
         def empty?

data/lib/logstash/outputs/qingstor/uploader.rb

@@ -10,6 +10,8 @@ module LogStash
   module Outputs
     class Qingstor
       class Uploader
+        require 'logstash/outputs/qingstor/multipart_uploader'
+
         TIME_BEFORE_RETRYING_SECONDS = 1
         DEFAULT_THREADPOOL = Concurrent::ThreadPoolExecutor.new(
           :min_thread => 1,
@@ -34,29 +36,38 @@ module LogStash
 
         def upload(file, options = {})
           upload_options = options.fetch(:upload_options, {})
+          upload_headers = process_encrypt_options(upload_options)
+
+          if file.size > 50 * 1024 * 1024
+            @logger.debug('multipart uploading file', :file => file.key)
+            multipart_uploader = MultipartUploader.new(@bucket, @logger, file, upload_headers)
+            multipart_uploader.upload
+          else
+            upload_headers['content_md5'] = Digest::MD5.file(file.path).to_s
+            upload_headers['body'] = ::File.read(file.path)
+            @logger.debug('uploading file', :file => file.key)
+            @bucket.put_object(file.key, upload_headers)
+          end
 
-          file_md5 = Digest::MD5.file(file.path).to_s
+          options[:on_complete].call(file) unless options[:on_complete].nil?
+        end
 
-          upload_headers = {
-            'content_md5' => file_md5,
-            'body' => ::File.read(file.path)
-          }
+        def process_encrypt_options(upload_options)
+          res = {}
 
           unless upload_options[:server_side_encryption_algorithm].nil?
             base64_key = Base64.strict_encode64(upload_options[:customer_key])
             key_md5 = Digest::MD5.hexdigest(upload_options[:customer_key])
             base64_key_md5 = Base64.strict_encode64(key_md5)
-            upload_headers.merge!(
+            res.merge!(
               'x_qs_encryption_customer_algorithm' =>
                 upload_options[:server_side_encryption_algorithm],
               'x_qs_encryption_customer_key' => base64_key,
               'x_qs_encryption_customer_key_md5' => base64_key_md5
             )
           end
-          @logger.debug('uploading file', :file => file.key)
-          bucket.put_object(file.key, upload_headers)
 
-          options[:on_complete].call(file) unless options[:on_complete].nil?
+          res
         end
 
         def stop

data/lib/logstash/outputs/qingstor.rb

@@ -256,11 +256,9 @@ class LogStash::Outputs::Qingstor < LogStash::Outputs::Base
        .each do |file|
       temp_file = TemporaryFile.create_from_existing_file(file,
                                                           temp_folder_path)
-
-      # Restoring too large file would cause java heap out of memory
-      if temp_file.size > 0 && temp_file.size < 100 * 1024 * 1024
-        temp_file.key = 'Restored/' + Time.new.strftime('%Y-%m-%d/')
-                        + temp_file.key
+      # Now multipart uploader supports file size up to 500GB
+      if temp_file.size > 0
+        temp_file.key = 'Restored/' + Time.new.strftime('%Y-%m-%d/') + temp_file.key
         @logger.debug('Recoving from crash and uploading',
                     :file => temp_file.key)
         @crash_uploader.upload_async(

data/logstash-output-qingstor.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-qingstor'
-  s.version       = '0.2.8'
+  s.version       = '0.3.1'
   s.licenses      = ['Apache License (2.0)']
   s.summary       = 'logstash output plugin for qingstor'
   s.description   = 'Collect the outputs of logstash and store into QingStor'

data/spec/outputs/qingstor_spec.rb

@@ -28,6 +28,10 @@ describe LogStash::Outputs::Qingstor do
     clean_remote_files
   end
 
+  before do
+    FileUtils.mkdir_p(tmpdir) unless File.exist?(tmpdir)
+  end
+
   it 'done work with minimal options' do
     fetch_event(options, events_and_encoded)
     expect(list_remote_file.size).to eq(1)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-qingstor
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.3.1
 platform: ruby
 authors:
 - Evan Zhao
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-12-26 00:00:00.000000000 Z
+date: 2018-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -111,6 +111,7 @@ files:
 - README_zh_CN.md
 - lib/logstash/outputs/qingstor.rb
 - lib/logstash/outputs/qingstor/file_repository.rb
+- lib/logstash/outputs/qingstor/multipart_uploader.rb
 - lib/logstash/outputs/qingstor/qingstor_validator.rb
 - lib/logstash/outputs/qingstor/rotation_policy.rb
 - lib/logstash/outputs/qingstor/temporary_file.rb