fluent-plugin-azurestorage 0.0.7 → 0.0.8

data/lib/fluent/plugin/upload_service.rb

@@ -1,208 +1,208 @@
-require 'pathname'
-require 'thread'
-
-module UploadService
-  MAX_BLOCK_SIZE = 4 * 1024 * 1024 # 4MB
-  MAX_PUT_SIZE = 64 * 1024 * 1024 # 64MB
-  THREAD_COUNT = 10
-
-  def self.extended(base)
-  end
-
-  def upload(source, options = {})
-    @thread_count = options[:thread_count] || THREAD_COUNT
-
-    size = File.size(source)
-
-    if size <= MAX_PUT_SIZE
-      content = File.open(source, 'rb') { |file| file.read }
-      self.create_block_blob(options[:container], options[:blob], content)
-    else
-      blocks = upload_blocks(source, options)
-      complete_upload(blocks, options)
-    end
-  end
-
-  def complete_upload(blocks, options)
-    options[:blob_content_type] = options[:content_type]
-
-    self.commit_blob_blocks(options[:container], options[:blob], blocks.map{ |block| [block[:block_id], :uncommitted] }, options)
-  end
-
-  def upload_blocks(source, options)
-    pending = BlockList.new(compute_blocks(source, options))
-    completed = BlockList.new
-    errors = upload_in_threads(pending, completed)
-    if errors.empty?
-      completed.to_a.sort_by { |block| block[:block_number] }
-    else
-      msg = "multipart upload failed: #{errors.map(&:message).join("; ")}"
-      raise BlockUploadError.new(msg, errors)
-    end
-  end
-
-  def compute_blocks(source, options)
-    size = File.size(source)
-    offset = 0
-    block_number = 1
-    blocks = []
-    while offset < size
-      blocks << {
-          container: options[:container],
-          blob: options[:blob],
-          block_id: block_number.to_s.rjust(5, '0'),
-          block_number: block_number,
-          body: FilePart.new(
-              source: source,
-              offset: offset,
-              size: block_size(size, MAX_BLOCK_SIZE, offset)
-          )
-      }
-      block_number += 1
-      offset += MAX_BLOCK_SIZE
-    end
-    blocks
-  end
-
-  def upload_in_threads(pending, completed)
-    threads = []
-    @thread_count.times do
-      thread = Thread.new do
-        begin
-          while block = pending.shift
-            content = block[:body].read
-            block[:body].close
-
-            options = {}
-            options[:content_md5] = Base64.strict_encode64(Digest::MD5.digest(content))
-            options[:timeout] = 30
-
-            content_md5 = self.create_blob_block(block[:container], block[:blob], block[:block_id], content, options)
-
-            if content_md5 != options[:content_md5]
-              raise "The block is corrupt: block = #{block[:block_id]}"
-            end
-
-            completed.push(block_id: block[:block_id], block_number: block[:block_number])
-          end
-          nil
-        rescue => error
-          # keep other threads from uploading other parts
-          pending.clear!
-          error
-        end
-      end
-      thread.abort_on_exception = true
-      threads << thread
-    end
-    threads.map(&:value).compact
-  end
-
-  def block_size(total_size, block_size, offset)
-    if offset + block_size > total_size
-      total_size - offset
-    else
-      block_size
-    end
-  end
-
-  # @api private
-  class BlockList
-
-    def initialize(blocks = [])
-      @blocks = blocks
-      @mutex = Mutex.new
-    end
-
-    def push(block)
-      @mutex.synchronize { @blocks.push(block) }
-    end
-
-    def shift
-      @mutex.synchronize { @blocks.shift }
-    end
-
-    def clear!
-      @mutex.synchronize { @blocks.clear }
-    end
-
-    def to_a
-      @mutex.synchronize { @blocks.dup }
-    end
-
-  end
-
-  class BlockUploadError < StandardError
-
-    def initialize(message, errors)
-      @errors = errors
-      super(message)
-    end
-
-    attr_reader :errors
-
-  end
-
-  class FilePart
-
-    def initialize(options = {})
-      @source = options[:source]
-      @first_byte = options[:offset]
-      @last_byte = @first_byte + options[:size]
-      @size = options[:size]
-      @file = nil
-    end
-
-    # @return [String,Pathname,File,Tempfile]
-    attr_reader :source
-
-    # @return [Integer]
-    attr_reader :first_byte
-
-    # @return [Integer]
-    attr_reader :last_byte
-
-    # @return [Integer]
-    attr_reader :size
-
-    def read(bytes = nil, output_buffer = nil)
-      open_file unless @file
-      read_from_file(bytes, output_buffer)
-    end
-
-    def rewind
-      if @file
-        @file.seek(@first_byte)
-        @position = @first_byte
-      end
-      0
-    end
-
-    def close
-      @file.close if @file
-    end
-
-    private
-
-    def open_file
-      @file = File.open(@source, 'rb')
-      rewind
-    end
-
-    def read_from_file(bytes, output_buffer)
-      if bytes
-        data = @file.read([remaining_bytes, bytes].min)
-        data = nil if data == ''
-      else
-        data = @file.read(remaining_bytes)
-      end
-      @position += data ? data.bytesize : 0
-      output_buffer ? output_buffer.replace(data || '') : data
-    end
-
-    def remaining_bytes
-      @last_byte - @position
-    end
-
-  end
+require 'pathname'
+require 'thread'
+
+module UploadService
+  MAX_BLOCK_SIZE = 4 * 1024 * 1024 # 4MB
+  MAX_PUT_SIZE = 64 * 1024 * 1024 # 64MB
+  THREAD_COUNT = 10
+
+  def self.extended(base)
+  end
+
+  def upload(source, options = {})
+    @thread_count = options[:thread_count] || THREAD_COUNT
+
+    size = File.size(source)
+
+    if size <= MAX_PUT_SIZE
+      content = File.open(source, 'rb') { |file| file.read }
+      self.create_block_blob(options[:container], options[:blob], content)
+    else
+      blocks = upload_blocks(source, options)
+      complete_upload(blocks, options)
+    end
+  end
+
+  def complete_upload(blocks, options)
+    options[:blob_content_type] = options[:content_type]
+
+    self.commit_blob_blocks(options[:container], options[:blob], blocks.map{ |block| [block[:block_id], :uncommitted] }, options)
+  end
+
+  def upload_blocks(source, options)
+    pending = BlockList.new(compute_blocks(source, options))
+    completed = BlockList.new
+    errors = upload_in_threads(pending, completed)
+    if errors.empty?
+      completed.to_a.sort_by { |block| block[:block_number] }
+    else
+      msg = "multipart upload failed: #{errors.map(&:message).join("; ")}"
+      raise BlockUploadError.new(msg, errors)
+    end
+  end
+
+  def compute_blocks(source, options)
+    size = File.size(source)
+    offset = 0
+    block_number = 1
+    blocks = []
+    while offset < size
+      blocks << {
+          container: options[:container],
+          blob: options[:blob],
+          block_id: block_number.to_s.rjust(5, '0'),
+          block_number: block_number,
+          body: FilePart.new(
+              source: source,
+              offset: offset,
+              size: block_size(size, MAX_BLOCK_SIZE, offset)
+          )
+      }
+      block_number += 1
+      offset += MAX_BLOCK_SIZE
+    end
+    blocks
+  end
+
+  def upload_in_threads(pending, completed)
+    threads = []
+    @thread_count.times do
+      thread = Thread.new do
+        begin
+          while block = pending.shift
+            content = block[:body].read
+            block[:body].close
+
+            options = {}
+            options[:content_md5] = Base64.strict_encode64(Digest::MD5.digest(content))
+            options[:timeout] = 30
+
+            content_md5 = self.create_blob_block(block[:container], block[:blob], block[:block_id], content, options)
+
+            if content_md5 != options[:content_md5]
+              raise "The block is corrupt: block = #{block[:block_id]}"
+            end
+
+            completed.push(block_id: block[:block_id], block_number: block[:block_number])
+          end
+          nil
+        rescue => error
+          # keep other threads from uploading other parts
+          pending.clear!
+          error
+        end
+      end
+      thread.abort_on_exception = true
+      threads << thread
+    end
+    threads.map(&:value).compact
+  end
+
+  def block_size(total_size, block_size, offset)
+    if offset + block_size > total_size
+      total_size - offset
+    else
+      block_size
+    end
+  end
+
+  # @api private
+  class BlockList
+
+    def initialize(blocks = [])
+      @blocks = blocks
+      @mutex = Mutex.new
+    end
+
+    def push(block)
+      @mutex.synchronize { @blocks.push(block) }
+    end
+
+    def shift
+      @mutex.synchronize { @blocks.shift }
+    end
+
+    def clear!
+      @mutex.synchronize { @blocks.clear }
+    end
+
+    def to_a
+      @mutex.synchronize { @blocks.dup }
+    end
+
+  end
+
+  class BlockUploadError < StandardError
+
+    def initialize(message, errors)
+      @errors = errors
+      super(message)
+    end
+
+    attr_reader :errors
+
+  end
+
+  class FilePart
+
+    def initialize(options = {})
+      @source = options[:source]
+      @first_byte = options[:offset]
+      @last_byte = @first_byte + options[:size]
+      @size = options[:size]
+      @file = nil
+    end
+
+    # @return [String,Pathname,File,Tempfile]
+    attr_reader :source
+
+    # @return [Integer]
+    attr_reader :first_byte
+
+    # @return [Integer]
+    attr_reader :last_byte
+
+    # @return [Integer]
+    attr_reader :size
+
+    def read(bytes = nil, output_buffer = nil)
+      open_file unless @file
+      read_from_file(bytes, output_buffer)
+    end
+
+    def rewind
+      if @file
+        @file.seek(@first_byte)
+        @position = @first_byte
+      end
+      0
+    end
+
+    def close
+      @file.close if @file
+    end
+
+    private
+
+    def open_file
+      @file = File.open(@source, 'rb')
+      rewind
+    end
+
+    def read_from_file(bytes, output_buffer)
+      if bytes
+        data = @file.read([remaining_bytes, bytes].min)
+        data = nil if data == ''
+      else
+        data = @file.read(remaining_bytes)
+      end
+      @position += data ? data.bytesize : 0
+      output_buffer ? output_buffer.replace(data || '') : data
+    end
+
+    def remaining_bytes
+      @last_byte - @position
+    end
+
+  end
 end

data/test/test_out_azurestorage.rb

@@ -1,224 +1,224 @@
-require 'fluent/test'
-require 'fluent/plugin/out_azurestorage'
-
-require 'test/unit/rr'
-require 'zlib'
-require 'fileutils'
-
-class AzureStorageOutputTest < Test::Unit::TestCase
-  def setup
-    require 'azure'
-    Fluent::Test.setup
-  end
-
-  CONFIG = %[
-    azure_storage_account test_storage_account
-    azure_storage_access_key dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=
-    azure_container test_container
-    path log
-    utc
-    buffer_type memory
-  ]
-
-  def create_driver(conf = CONFIG)
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::AzureStorageOutput) do
-      def write(chunk)
-        chunk.read
-      end
-
-      private
-
-      def ensure_container
-      end
-
-    end.configure(conf)
-  end
-
-  def test_configure
-    d = create_driver
-    assert_equal 'test_storage_account', d.instance.azure_storage_account
-    assert_equal 'dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=', d.instance.azure_storage_access_key
-    assert_equal 'test_container', d.instance.azure_container
-    assert_equal 'log', d.instance.path
-    assert_equal 'gz', d.instance.instance_variable_get(:@compressor).ext
-    assert_equal 'application/x-gzip', d.instance.instance_variable_get(:@compressor).content_type
-  end
-
-  def test_configure_with_mime_type_json
-    conf = CONFIG.clone
-    conf << "\nstore_as json\n"
-    d = create_driver(conf)
-    assert_equal 'json', d.instance.instance_variable_get(:@compressor).ext
-    assert_equal 'application/json', d.instance.instance_variable_get(:@compressor).content_type
-  end
-
-  def test_configure_with_mime_type_text
-    conf = CONFIG.clone
-    conf << "\nstore_as text\n"
-    d = create_driver(conf)
-    assert_equal 'txt', d.instance.instance_variable_get(:@compressor).ext
-    assert_equal 'text/plain', d.instance.instance_variable_get(:@compressor).content_type
-  end
-
-  def test_configure_with_mime_type_lzo
-    conf = CONFIG.clone
-    conf << "\nstore_as lzo\n"
-    d = create_driver(conf)
-    assert_equal 'lzo', d.instance.instance_variable_get(:@compressor).ext
-    assert_equal 'application/x-lzop', d.instance.instance_variable_get(:@compressor).content_type
-  rescue => e
-    # TODO: replace code with disable lzop command
-    assert(e.is_a?(Fluent::ConfigError))
-  end
-
-  def test_path_slicing
-    config = CONFIG.clone.gsub(/path\slog/, "path log/%Y/%m/%d")
-    d = create_driver(config)
-    path_slicer = d.instance.instance_variable_get(:@path_slicer)
-    path = d.instance.instance_variable_get(:@path)
-    slice = path_slicer.call(path)
-    assert_equal slice, Time.now.utc.strftime("log/%Y/%m/%d")
-  end
-
-  def test_path_slicing_utc
-    config = CONFIG.clone.gsub(/path\slog/, "path log/%Y/%m/%d")
-    config << "\nutc\n"
-    d = create_driver(config)
-    path_slicer = d.instance.instance_variable_get(:@path_slicer)
-    path = d.instance.instance_variable_get(:@path)
-    slice = path_slicer.call(path)
-    assert_equal slice, Time.now.utc.strftime("log/%Y/%m/%d")
-  end
-
-  def test_format
-    d = create_driver
-
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
-
-    d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
-    d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
-
-    d.run
-  end
-
-  def test_format_included_tag_and_time
-    config = [CONFIG, 'include_tag_key true', 'include_time_key true'].join("\n")
-    d = create_driver(config)
-
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
-
-    d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
-    d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
-
-    d.run
-  end
-
-  def test_format_with_format_ltsv
-    config = [CONFIG, 'format ltsv'].join("\n")
-    d = create_driver(config)
-
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1, "b"=>1}, time)
-    d.emit({"a"=>2, "b"=>2}, time)
-
-    d.expect_format %[a:1\tb:1\n]
-    d.expect_format %[a:2\tb:2\n]
-
-    d.run
-  end
-
-  def test_format_with_format_json
-    config = [CONFIG, 'format json'].join("\n")
-    d = create_driver(config)
-
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
-
-    d.expect_format %[{"a":1}\n]
-    d.expect_format %[{"a":2}\n]
-
-    d.run
-  end
-
-  def test_format_with_format_json_included_tag
-    config = [CONFIG, 'format json', 'include_tag_key true'].join("\n")
-    d = create_driver(config)
-
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
-
-    d.expect_format %[{"a":1,"tag":"test"}\n]
-    d.expect_format %[{"a":2,"tag":"test"}\n]
-
-    d.run
-  end
-
-  def test_format_with_format_json_included_time
-    config = [CONFIG, 'format json', 'include_time_key true'].join("\n")
-    d = create_driver(config)
-
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
-
-    d.expect_format %[{"a":1,"time":"2011-01-02T13:14:15Z"}\n]
-    d.expect_format %[{"a":2,"time":"2011-01-02T13:14:15Z"}\n]
-
-    d.run
-  end
-
-  def test_format_with_format_json_included_tag_and_time
-    config = [CONFIG, 'format json', 'include_tag_key true', 'include_time_key true'].join("\n")
-    d = create_driver(config)
-
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
-
-    d.expect_format %[{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
-    d.expect_format %[{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
-
-    d.run
-  end
-
-  def test_chunk_to_write
-    d = create_driver
-
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
-
-    # AzureStorageOutputTest#write returns chunk.read
-    data = d.run
-
-    assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
-                 %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
-                 data
-  end
-
-  CONFIG_TIME_SLICE = %[
-    hostname testing.node.local
-    azure_storage_account test_storage_account
-    azure_storage_access_key dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=
-    azure_container test_container
-    azure_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostname}.%{file_extension}
-    time_slice_format %Y%m%d-%H
-    path log
-    utc
-    buffer_type memory
-    log_level debug
-  ]
-
-  def create_time_sliced_driver(conf = CONFIG_TIME_SLICE)
-    d = Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::AzureStorageOutput) do
-    end.configure(conf)
-    d
-  end
-
-end
+require 'fluent/test'
+require 'fluent/plugin/out_azurestorage'
+
+require 'test/unit/rr'
+require 'zlib'
+require 'fileutils'
+
+class AzureStorageOutputTest < Test::Unit::TestCase
+  def setup
+    require 'azure'
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    azure_storage_account test_storage_account
+    azure_storage_access_key dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=
+    azure_container test_container
+    path log
+    utc
+    buffer_type memory
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::AzureStorageOutput) do
+      def write(chunk)
+        chunk.read
+      end
+
+      private
+
+      def ensure_container
+      end
+
+    end.configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal 'test_storage_account', d.instance.azure_storage_account
+    assert_equal 'dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=', d.instance.azure_storage_access_key
+    assert_equal 'test_container', d.instance.azure_container
+    assert_equal 'log', d.instance.path
+    assert_equal 'gz', d.instance.instance_variable_get(:@compressor).ext
+    assert_equal 'application/x-gzip', d.instance.instance_variable_get(:@compressor).content_type
+  end
+
+  def test_configure_with_mime_type_json
+    conf = CONFIG.clone
+    conf << "\nstore_as json\n"
+    d = create_driver(conf)
+    assert_equal 'json', d.instance.instance_variable_get(:@compressor).ext
+    assert_equal 'application/json', d.instance.instance_variable_get(:@compressor).content_type
+  end
+
+  def test_configure_with_mime_type_text
+    conf = CONFIG.clone
+    conf << "\nstore_as text\n"
+    d = create_driver(conf)
+    assert_equal 'txt', d.instance.instance_variable_get(:@compressor).ext
+    assert_equal 'text/plain', d.instance.instance_variable_get(:@compressor).content_type
+  end
+
+  def test_configure_with_mime_type_lzo
+    conf = CONFIG.clone
+    conf << "\nstore_as lzo\n"
+    d = create_driver(conf)
+    assert_equal 'lzo', d.instance.instance_variable_get(:@compressor).ext
+    assert_equal 'application/x-lzop', d.instance.instance_variable_get(:@compressor).content_type
+  rescue => e
+    # TODO: replace code with disable lzop command
+    assert(e.is_a?(Fluent::ConfigError))
+  end
+
+  def test_path_slicing
+    config = CONFIG.clone.gsub(/path\slog/, "path log/%Y/%m/%d")
+    d = create_driver(config)
+    path_slicer = d.instance.instance_variable_get(:@path_slicer)
+    path = d.instance.instance_variable_get(:@path)
+    slice = path_slicer.call(path)
+    assert_equal slice, Time.now.utc.strftime("log/%Y/%m/%d")
+  end
+
+  def test_path_slicing_utc
+    config = CONFIG.clone.gsub(/path\slog/, "path log/%Y/%m/%d")
+    config << "\nutc\n"
+    d = create_driver(config)
+    path_slicer = d.instance.instance_variable_get(:@path_slicer)
+    path = d.instance.instance_variable_get(:@path)
+    slice = path_slicer.call(path)
+    assert_equal slice, Time.now.utc.strftime("log/%Y/%m/%d")
+  end
+
+  def test_format
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
+    d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
+
+    d.run
+  end
+
+  def test_format_included_tag_and_time
+    config = [CONFIG, 'include_tag_key true', 'include_time_key true'].join("\n")
+    d = create_driver(config)
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
+    d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
+
+    d.run
+  end
+
+  def test_format_with_format_ltsv
+    config = [CONFIG, 'format ltsv'].join("\n")
+    d = create_driver(config)
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1, "b"=>1}, time)
+    d.emit({"a"=>2, "b"=>2}, time)
+
+    d.expect_format %[a:1\tb:1\n]
+    d.expect_format %[a:2\tb:2\n]
+
+    d.run
+  end
+
+  def test_format_with_format_json
+    config = [CONFIG, 'format json'].join("\n")
+    d = create_driver(config)
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    d.expect_format %[{"a":1}\n]
+    d.expect_format %[{"a":2}\n]
+
+    d.run
+  end
+
+  def test_format_with_format_json_included_tag
+    config = [CONFIG, 'format json', 'include_tag_key true'].join("\n")
+    d = create_driver(config)
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    d.expect_format %[{"a":1,"tag":"test"}\n]
+    d.expect_format %[{"a":2,"tag":"test"}\n]
+
+    d.run
+  end
+
+  def test_format_with_format_json_included_time
+    config = [CONFIG, 'format json', 'include_time_key true'].join("\n")
+    d = create_driver(config)
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    d.expect_format %[{"a":1,"time":"2011-01-02T13:14:15Z"}\n]
+    d.expect_format %[{"a":2,"time":"2011-01-02T13:14:15Z"}\n]
+
+    d.run
+  end
+
+  def test_format_with_format_json_included_tag_and_time
+    config = [CONFIG, 'format json', 'include_tag_key true', 'include_time_key true'].join("\n")
+    d = create_driver(config)
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    d.expect_format %[{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
+    d.expect_format %[{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
+
+    d.run
+  end
+
+  def test_chunk_to_write
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    # AzureStorageOutputTest#write returns chunk.read
+    data = d.run
+
+    assert_equal [%[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
+                 %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]],
+                 data
+  end
+
+  CONFIG_TIME_SLICE = %[
+    hostname testing.node.local
+    azure_storage_account test_storage_account
+    azure_storage_access_key dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=
+    azure_container test_container
+    azure_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostname}.%{file_extension}
+    time_slice_format %Y%m%d-%H
+    path log
+    utc
+    buffer_type memory
+    log_level debug
+  ]
+
+  def create_time_sliced_driver(conf = CONFIG_TIME_SLICE)
+    d = Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::AzureStorageOutput) do
+    end.configure(conf)
+    d
+  end
+
+end