fluent-plugin-azurestorage 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8bb81cd14c71e59c606798aacf0fcbf3816d65c8
-  data.tar.gz: b520ba98690443f89e9f683248ba321675de0ffa
+  metadata.gz: 920277cb0b8ab63f16a0626a8c1e9ea341e62b28
+  data.tar.gz: 0a86562c7713af3c45f8863a503e29bdb5aed1cc
 SHA512:
-  metadata.gz: 98704f9ebff04476910ff2292e141fd8bf11f6831a27d429cdcecf16a8a2053bee071dbb2c8310b7ed600a42d71d7ddf52111b019b4cf968ef6704eda3de0113
-  data.tar.gz: 06eac631498041b2c523b926587afc49dd90d13c39ba92a527beb8e04ea9369449885ab2dc0a055282af090491c7b5ca3fe8a016bc2569a083628d99e815ae39
+  metadata.gz: 61f809feb8edb5810251a3ee39daeb6a169b4178e932bb02c7ceadc821755ac0bfb20c0a4cc2b3ea4ba4e7e77606c8cccb4b06aafc64cbc0cc4cf07dc65f25a4
+  data.tar.gz: 477d61ae9f6faad494a1ef3d169929ad4416fa09ca7af21c450f70e0eaa70b1c3e85b9d3cff68dfa431ef204dac3216165e7691704bc571da99f0ce6a3c461d1

data/README.md

@@ -1,5 +1,7 @@
 # Azure Storage output plugin for Fluentd
 
+[![Build Status](https://travis-ci.org/htgc/fluent-plugin-azurestorage.svg?branch=master)](https://travis-ci.org/htgc/fluent-plugin-azurestorage)
+
 ## Overview
 
 Azure Storate output plugin buffers logs in local file and upload them to Azure Storage periodically.

data/VERSION

@@ -1 +1 @@
-0.0.6
+0.0.7

data/fluent-plugin-azurestorage.gemspec

@@ -21,4 +21,6 @@ Gem::Specification.new do |gem|
   gem.add_dependency "azure", "0.6.4"
   gem.add_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
   gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "test-unit", ">= 3.0.8"
+  gem.add_development_dependency "test-unit-rr", ">= 1.0.3"
 end

data/lib/fluent/plugin/out_azurestorage.rb

@@ -7,6 +7,7 @@ module Fluent
     def initialize
       super
       require 'azure'
+      require 'fluent/plugin/upload_service'
       require 'zlib'
       require 'time'
       require 'tempfile'
@@ -81,6 +82,7 @@ module Fluent
         end
       end
       @bs = Azure::BlobService.new
+      @bs.extend UploadService
 
       ensure_container
     end
@@ -117,8 +119,13 @@ module Fluent
       begin
         @compressor.compress(chunk, tmp)
         tmp.close
-        content = File.open(tmp.path, 'rb') { |file| file.read }
-        @bs.create_block_blob(@azure_container, storage_path, content)
+
+        options = {}
+        options[:content_type] = @compressor.content_type
+        options[:container] = @azure_container
+        options[:blob] = storage_path
+
+        @bs.upload(tmp.path, options)
       end
     end
 

data/lib/fluent/plugin/upload_service.rb

@@ -0,0 +1,208 @@
+require 'pathname'
+require 'thread'
+
+module UploadService
+  MAX_BLOCK_SIZE = 4 * 1024 * 1024 # 4MB
+  MAX_PUT_SIZE = 64 * 1024 * 1024 # 64MB
+  THREAD_COUNT = 10
+
+  def self.extended(base)
+  end
+
+  def upload(source, options = {})
+    @thread_count = options[:thread_count] || THREAD_COUNT
+
+    size = File.size(source)
+
+    if size <= MAX_PUT_SIZE
+      content = File.open(source, 'rb') { |file| file.read }
+      self.create_block_blob(options[:container], options[:blob], content)
+    else
+      blocks = upload_blocks(source, options)
+      complete_upload(blocks, options)
+    end
+  end
+
+  def complete_upload(blocks, options)
+    options[:blob_content_type] = options[:content_type]
+
+    self.commit_blob_blocks(options[:container], options[:blob], blocks.map{ |block| [block[:block_id], :uncommitted] }, options)
+  end
+
+  def upload_blocks(source, options)
+    pending = BlockList.new(compute_blocks(source, options))
+    completed = BlockList.new
+    errors = upload_in_threads(pending, completed)
+    if errors.empty?
+      completed.to_a.sort_by { |block| block[:block_number] }
+    else
+      msg = "multipart upload failed: #{errors.map(&:message).join("; ")}"
+      raise BlockUploadError.new(msg, errors)
+    end
+  end
+
+  def compute_blocks(source, options)
+    size = File.size(source)
+    offset = 0
+    block_number = 1
+    blocks = []
+    while offset < size
+      blocks << {
+          container: options[:container],
+          blob: options[:blob],
+          block_id: block_number.to_s.rjust(5, '0'),
+          block_number: block_number,
+          body: FilePart.new(
+              source: source,
+              offset: offset,
+              size: block_size(size, MAX_BLOCK_SIZE, offset)
+          )
+      }
+      block_number += 1
+      offset += MAX_BLOCK_SIZE
+    end
+    blocks
+  end
+
+  def upload_in_threads(pending, completed)
+    threads = []
+    @thread_count.times do
+      thread = Thread.new do
+        begin
+          while block = pending.shift
+            content = block[:body].read
+            block[:body].close
+
+            options = {}
+            options[:content_md5] = Base64.strict_encode64(Digest::MD5.digest(content))
+            options[:timeout] = 30
+
+            content_md5 = self.create_blob_block(block[:container], block[:blob], block[:block_id], content, options)
+
+            if content_md5 != options[:content_md5]
+              raise "The block is corrupt: block = #{block[:block_id]}"
+            end
+
+            completed.push(block_id: block[:block_id], block_number: block[:block_number])
+          end
+          nil
+        rescue => error
+          # keep other threads from uploading other parts
+          pending.clear!
+          error
+        end
+      end
+      thread.abort_on_exception = true
+      threads << thread
+    end
+    threads.map(&:value).compact
+  end
+
+  def block_size(total_size, block_size, offset)
+    if offset + block_size > total_size
+      total_size - offset
+    else
+      block_size
+    end
+  end
+
+  # @api private
+  class BlockList
+
+    def initialize(blocks = [])
+      @blocks = blocks
+      @mutex = Mutex.new
+    end
+
+    def push(block)
+      @mutex.synchronize { @blocks.push(block) }
+    end
+
+    def shift
+      @mutex.synchronize { @blocks.shift }
+    end
+
+    def clear!
+      @mutex.synchronize { @blocks.clear }
+    end
+
+    def to_a
+      @mutex.synchronize { @blocks.dup }
+    end
+
+  end
+
+  class BlockUploadError < StandardError
+
+    def initialize(message, errors)
+      @errors = errors
+      super(message)
+    end
+
+    attr_reader :errors
+
+  end
+
+  class FilePart
+
+    def initialize(options = {})
+      @source = options[:source]
+      @first_byte = options[:offset]
+      @last_byte = @first_byte + options[:size]
+      @size = options[:size]
+      @file = nil
+    end
+
+    # @return [String,Pathname,File,Tempfile]
+    attr_reader :source
+
+    # @return [Integer]
+    attr_reader :first_byte
+
+    # @return [Integer]
+    attr_reader :last_byte
+
+    # @return [Integer]
+    attr_reader :size
+
+    def read(bytes = nil, output_buffer = nil)
+      open_file unless @file
+      read_from_file(bytes, output_buffer)
+    end
+
+    def rewind
+      if @file
+        @file.seek(@first_byte)
+        @position = @first_byte
+      end
+      0
+    end
+
+    def close
+      @file.close if @file
+    end
+
+    private
+
+    def open_file
+      @file = File.open(@source, 'rb')
+      rewind
+    end
+
+    def read_from_file(bytes, output_buffer)
+      if bytes
+        data = @file.read([remaining_bytes, bytes].min)
+        data = nil if data == ''
+      else
+        data = @file.read(remaining_bytes)
+      end
+      @position += data ? data.bytesize : 0
+      output_buffer ? output_buffer.replace(data || '') : data
+    end
+
+    def remaining_bytes
+      @last_byte - @position
+    end
+
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azurestorage
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Hidemasa Togashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-09 00:00:00.000000000 Z
+date: 2015-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -72,6 +72,34 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.8
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.8
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.3
 description: Azure Storage output plugin for Fluentd event collector
 email:
 - togachiro@gmail.com
@@ -91,6 +119,7 @@ files:
 - lib/fluent/plugin/azurestorage_compressor_lzma2.rb
 - lib/fluent/plugin/azurestorage_compressor_lzo.rb
 - lib/fluent/plugin/out_azurestorage.rb
+- lib/fluent/plugin/upload_service.rb
 - test/test_out_azurestorage.rb
 homepage: https://github.com/htgc/fluent-plugin-azurestorage
 licenses: