fluentd 0.14.6 → 0.14.7

data/test/helper.rb

@@ -44,7 +44,6 @@ require 'fluent/log'
 require 'fluent/test'
 require 'fluent/test/helpers'
 require 'fluent/plugin/base'
-require 'fluent/log'
 require 'fluent/plugin_id'
 require 'fluent/plugin_helper'
 require 'fluent/msgpack_factory'

data/test/plugin/test_buffer.rb

@@ -1,6 +1,8 @@
 require_relative '../helper'
 require 'fluent/plugin/buffer'
 require 'fluent/plugin/buffer/memory_chunk'
+require 'fluent/plugin/compressable'
+require 'fluent/plugin/buffer/chunk'
 require 'fluent/event'
 require 'flexmock/test_unit'
 
@@ -18,7 +20,7 @@ module FluentPluginBufferTest
   class DummyMemoryChunk < Fluent::Plugin::Buffer::MemoryChunk
     attr_reader :append_count, :rollbacked, :closed, :purged
     attr_accessor :failing
-    def initialize(metadata)
+    def initialize(metadata, compress: :text)
       super
       @append_count = 0
       @rollbacked = false
@@ -77,7 +79,7 @@ module FluentPluginBufferTest
       return staged, queued
     end
     def generate_chunk(metadata)
-      DummyMemoryChunk.new(metadata)
+      DummyMemoryChunk.new(metadata, compress: @compress)
     end
   end
 end
@@ -865,6 +867,10 @@ class BufferTest < Test::Unit::TestCase
       assert{ @p.stage[@dm1].size == 2 }
       assert @p.stage[@dm1].rollbacked
     end
+
+    test '#compress returns :text' do
+      assert_equal :text, @p.compress
+    end
   end
 
   sub_test_case 'standard format with configuration for test with lower chunk limit size' do
@@ -1197,4 +1203,18 @@ class BufferTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case 'when compress is gzip' do
+    setup do
+      @p = create_buffer({'compress' => 'gzip'})
+    end
+
+    test '#compress returns :gzip' do
+      assert_equal :gzip, @p.compress
+    end
+
+    test 'create decompressable chunk' do
+      chunk = @p.generate_chunk(create_metadata)
+      assert chunk.singleton_class.ancestors.include?(Fluent::Plugin::Buffer::Chunk::Decompressable)
+    end
+  end
 end

data/test/plugin/test_buffer_chunk.rb

@@ -31,7 +31,6 @@ class BufferChunkTest < Test::Unit::TestCase
       assert chunk.respond_to?(:read)
       assert chunk.respond_to?(:open)
       assert chunk.respond_to?(:write_to)
-      assert chunk.respond_to?(:msgpack_each)
       assert_raise(NotImplementedError){ chunk.append([]) }
       assert_raise(NotImplementedError){ chunk.concat(nil, 0) }
       assert_raise(NotImplementedError){ chunk.commit }
@@ -43,7 +42,29 @@ class BufferChunkTest < Test::Unit::TestCase
       assert_raise(NotImplementedError){ chunk.read }
       assert_raise(NotImplementedError){ chunk.open(){} }
       assert_raise(NotImplementedError){ chunk.write_to(nil) }
-      assert_raise(NotImplementedError){ chunk.msgpack_each(){|v| v} }
+      assert !chunk.respond_to?(:msgpack_each)
+    end
+
+    test 'has method #each and #msgpack_each only when extended by ChunkMessagePackEventStreamer' do
+      meta = Object.new
+      chunk = Fluent::Plugin::Buffer::Chunk.new(meta)
+
+      assert !chunk.respond_to?(:each)
+      assert !chunk.respond_to?(:msgpack_each)
+
+      chunk.extend Fluent::ChunkMessagePackEventStreamer
+      assert chunk.respond_to?(:each)
+      assert chunk.respond_to?(:msgpack_each)
+    end
+
+    test 'some methods raise ArgumentError with an option of `compressed: :gzip` and without extending Compressble`' do
+      meta = Object.new
+      chunk = Fluent::Plugin::Buffer::Chunk.new(meta)
+
+      assert_raise(ArgumentError){ chunk.read(compressed: :gzip) }
+      assert_raise(ArgumentError){ chunk.open(compressed: :gzip){} }
+      assert_raise(ArgumentError){ chunk.write_to(nil, compressed: :gzip) }
+      assert_raise(ArgumentError){ chunk.append(nil, compress: :gzip) }
     end
   end
 
@@ -56,7 +77,7 @@ class BufferChunkTest < Test::Unit::TestCase
     def size
       @data.size
     end
-    def open
+    def open(**kwargs)
       require 'stringio'
       io = StringIO.new(@data)
       yield io
@@ -152,9 +173,10 @@ class BufferChunkTest < Test::Unit::TestCase
       assert "my data\nyour data\n", io.to_s
     end
 
-    test 'can feed objects into blocks with unpacking msgpack' do
+    test 'can feed objects into blocks with unpacking msgpack if ChunkMessagePackEventStreamer is included' do
       require 'msgpack'
       c = TestChunk.new(Object.new)
+      c.extend Fluent::ChunkMessagePackEventStreamer
       c.data << MessagePack.pack(['my data', 1])
       c.data << MessagePack.pack(['your data', 2])
       ary = []
@@ -165,4 +187,12 @@ class BufferChunkTest < Test::Unit::TestCase
       assert_equal ['your data', 2], ary[1]
     end
   end
+
+  sub_test_case 'when compress is gzip' do
+    test 'create decompressable chunk' do
+      meta = Object.new
+      chunk = Fluent::Plugin::Buffer::Chunk.new(meta, compress: :gzip)
+      assert chunk.singleton_class.ancestors.include?(Fluent::Plugin::Buffer::Chunk::Decompressable)
+    end
+  end
 end

data/test/plugin/test_buffer_file_chunk.rb

@@ -1,5 +1,6 @@
 require_relative '../helper'
 require 'fluent/plugin/buffer/file_chunk'
+require 'fluent/plugin/compressable'
 require 'fluent/unique_id'
 
 require 'fileutils'
@@ -8,6 +9,8 @@ require 'time'
 require 'timecop'
 
 class BufferFileChunkTest < Test::Unit::TestCase
+  include Fluent::Plugin::Compressable
+
   setup do
     @klass = Fluent::Plugin::Buffer::FileChunk
     @chunkdir = File.expand_path('../../tmp/buffer_file_chunk', __FILE__)
@@ -768,4 +771,74 @@ class BufferFileChunkTest < Test::Unit::TestCase
       end
     end
   end
+
+  sub_test_case 'compressed buffer' do
+    setup do
+      @src = 'text data for compressing' * 5
+      @gzipped_src = compress(@src)
+    end
+
+    test '#append with compress option writes  compressed data to chunk when compress is gzip' do
+      c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip)
+      c.append([@src, @src], compress: :gzip)
+      c.commit
+
+      # check chunk is compressed
+      assert c.read(compressed: :gzip).size < [@src, @src].join("").size
+
+      assert_equal @src + @src, c.read
+    end
+
+    test '#open passes io object having decompressed data to a block when compress is gzip' do
+      c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip)
+      c.concat(@gzipped_src, @src.size)
+      c.commit
+
+      decomressed_data = c.open do |io|
+        v = io.read
+        assert_equal @src, v
+        v
+      end
+      assert_equal @src, decomressed_data
+    end
+
+    test '#open with compressed option passes io object having decompressed data to a block when compress is gzip' do
+      c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip)
+      c.concat(@gzipped_src, @src.size)
+      c.commit
+
+      comressed_data = c.open(compressed: :gzip) do |io|
+        v = io.read
+        assert_equal @gzipped_src, v
+        v
+      end
+      assert_equal @gzipped_src, comressed_data
+    end
+
+    test '#write_to writes decompressed data when compress is gzip' do
+      c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip)
+      c.concat(@gzipped_src, @src.size)
+      c.commit
+
+      assert_equal @src, c.read
+      assert_equal @gzipped_src, c.read(compressed: :gzip)
+
+      io = StringIO.new
+      c.write_to(io)
+      assert_equal @src, io.string
+    end
+
+    test '#write_to with compressed option writes compressed data when compress is gzip' do
+      c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :gzip)
+      c.concat(@gzipped_src, @src.size)
+      c.commit
+
+      assert_equal @src, c.read
+      assert_equal @gzipped_src, c.read(compressed: :gzip)
+
+      io = StringIO.new
+      c.write_to(io, compressed: :gzip)
+      assert_equal @gzipped_src, io.string
+    end
+  end
 end

data/test/plugin/test_buffer_memory_chunk.rb

@@ -1,9 +1,12 @@
 require_relative '../helper'
 require 'fluent/plugin/buffer/memory_chunk'
+require 'fluent/plugin/compressable'
 
 require 'json'
 
 class BufferMemoryChunkTest < Test::Unit::TestCase
+  include Fluent::Plugin::Compressable
+
   setup do
     @c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new)
   end
@@ -262,4 +265,74 @@ class BufferMemoryChunkTest < Test::Unit::TestCase
     assert_equal d3.to_json + "\n", lines[2]
     assert_equal d4.to_json + "\n", lines[3]
   end
+
+  sub_test_case 'compressed buffer' do
+    setup do
+      @src = 'text data for compressing' * 5
+      @gzipped_src = compress(@src)
+    end
+
+    test '#append with compress option writes  compressed data to chunk when compress is gzip' do
+      c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :gzip)
+      c.append([@src, @src], compress: :gzip)
+      c.commit
+
+      # check chunk is compressed
+      assert c.read(compressed: :gzip).size < [@src, @src].join("").size
+
+      assert_equal @src + @src, c.read
+    end
+
+    test '#open passes io object having decompressed data to a block when compress is gzip' do
+      c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :gzip)
+      c.concat(@gzipped_src, @src.size)
+      c.commit
+
+      decomressed_data = c.open do |io|
+        v = io.read
+        assert_equal @src, v
+        v
+      end
+      assert_equal @src, decomressed_data
+    end
+
+    test '#open with compressed option passes io object having decompressed data to a block when compress is gzip' do
+      c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :gzip)
+      c.concat(@gzipped_src, @src.size)
+      c.commit
+
+      comressed_data = c.open(compressed: :gzip) do |io|
+        v = io.read
+        assert_equal @gzipped_src, v
+        v
+      end
+      assert_equal @gzipped_src, comressed_data
+    end
+
+    test '#write_to writes decompressed data when compress is gzip' do
+      c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :gzip)
+      c.concat(@gzipped_src, @src.size)
+      c.commit
+
+      assert_equal @src, c.read
+      assert_equal @gzipped_src, c.read(compressed: :gzip)
+
+      io = StringIO.new
+      c.write_to(io)
+      assert_equal @src, io.string
+    end
+
+    test '#write_to with compressed option writes compressed data when compress is gzip' do
+      c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :gzip)
+      c.concat(@gzipped_src, @src.size)
+      c.commit
+
+      assert_equal @src, c.read
+      assert_equal @gzipped_src, c.read(compressed: :gzip)
+
+      io = StringIO.new
+      c.write_to(io, compressed: :gzip)
+      assert_equal @gzipped_src, io.string
+    end
+  end
 end

data/test/plugin/test_compressable.rb

@@ -0,0 +1,81 @@
+require_relative '../helper'
+require 'fluent/plugin/compressable'
+
+class CompressableTest < Test::Unit::TestCase
+  include Fluent::Plugin::Compressable
+
+  sub_test_case '#compress' do
+    setup do
+      @src = 'text data for compressing' * 5
+      @gzipped_src = compress(@src)
+    end
+
+    test 'compress data' do
+      assert compress(@src).size < @src.size
+      assert_not_equal @gzipped_src, @src
+    end
+
+    test 'write compressed data to IO with output_io option' do
+      io = StringIO.new
+      compress(@src, output_io: io)
+      assert_equal @gzipped_src, io.string
+    end
+  end
+
+  sub_test_case '#decompress' do
+    setup do
+      @src = 'text data for compressing' * 5
+      @gzipped_src = compress(@src)
+    end
+
+    test 'decompress compressed data' do
+      assert_equal @src, decompress(@gzipped_src)
+    end
+
+    test 'write decompressed data to IO with output_io option' do
+      io = StringIO.new
+      decompress(@gzipped_src, output_io: io)
+      assert_equal @src, io.string
+    end
+
+    test 'return decompressed string with output_io option' do
+      io = StringIO.new(@gzipped_src)
+      assert_equal @src, decompress(input_io: io)
+    end
+
+    test 'decompress multiple compressed data' do
+      src1 = 'text data'
+      src2 = 'text data2'
+      gzipped_src = compress(src1) + compress(src2)
+
+      assert_equal src1 + src2, decompress(gzipped_src)
+    end
+
+    test 'decompress with input_io and output_io' do
+      input_io = StringIO.new(@gzipped_src)
+      output_io = StringIO.new
+
+      decompress(input_io: input_io, output_io: output_io)
+      assert_equal @src, output_io.string
+    end
+
+    test 'decompress multiple compressed data with input_io and output_io' do
+      src1 = 'text data'
+      src2 = 'text data2'
+      gzipped_src = compress(src1) + compress(src2)
+
+      input_io = StringIO.new(gzipped_src)
+      output_io = StringIO.new
+
+      decompress(input_io: input_io, output_io: output_io)
+      assert_equal src1 + src2, output_io.string
+    end
+
+    test 'return the received value as it is with empty string or nil' do
+      assert_equal nil, decompress
+      assert_equal nil, decompress(nil)
+      assert_equal '', decompress('')
+      assert_equal '', decompress('', output_io: StringIO.new)
+    end
+  end
+end

data/test/plugin/test_formatter_json.rb

@@ -1,4 +1,5 @@
 require_relative '../helper'
+require 'json'
 require 'fluent/test/driver/formatter'
 require 'fluent/plugin/formatter_json'
 
@@ -20,11 +21,23 @@ class JsonFormatterTest < ::Test::Unit::TestCase
     {'message' => 'awesome'}
   end
 
+  def symbolic_record
+    {:message => :awesome}
+  end
+
   data('oj' => 'oj', 'yajl' => 'yajl')
   def test_format(data)
     d = create_driver('json_parser' => data)
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("#{Yajl.dump(record)}\n", formatted)
+    assert_equal("#{JSON.generate(record)}\n", formatted)
+  end
+
+  data('oj' => 'oj', 'yajl' => 'yajl')
+  def test_format_with_symbolic_record(data)
+    d = create_driver('json_parser' => data)
+    formatted = d.instance.format(tag, @time, symbolic_record)
+
+    assert_equal("#{JSON.generate(record)}\n", formatted)
   end
 end

data/test/plugin/test_in_forward.rb

@@ -5,9 +5,13 @@ require 'fluent/test/startup_shutdown'
 require 'base64'
 
 require 'fluent/env'
+require 'fluent/event'
 require 'fluent/plugin/in_forward'
+require 'fluent/plugin/compressable'
 
 class ForwardInputTest < Test::Unit::TestCase
+ include Fluent::Plugin::Compressable
+
   def setup
     Fluent::Test.setup
     @responses = []  # for testing responses after sending data
@@ -465,6 +469,66 @@ class ForwardInputTest < Test::Unit::TestCase
     end
   end
 
+  class CompressedPackedForward < self
+    extend Fluent::Test::StartupShutdown
+
+    def test_set_compress_to_option
+      d = create_driver
+
+      time = event_time("2011-01-02 13:14:15 UTC").to_i
+      events = [
+        ["tag1", time, {"a"=>1}],
+        ["tag1", time, {"a"=>2}]
+      ]
+
+      # create compressed entries
+      entries = ''
+      events.each do |_tag, _time, record|
+        v = [_time, record].to_msgpack
+        entries << compress(v)
+      end
+      chunk = ["tag1", entries, { 'compressed' => 'gzip' }].to_msgpack
+
+      d.run do
+        Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
+          option = d.instance.send(:on_message, obj, chunk.size, PEERADDR)
+          assert_equal 'gzip', option['compressed']
+        end
+      end
+
+      assert_equal events, d.emits
+    end
+
+    def test_create_CompressedMessagePackEventStream_with_gzip_compress_option
+      d = create_driver
+
+      time = event_time("2011-01-02 13:14:15 UTC").to_i
+      events = [
+        ["tag1", time, {"a"=>1}],
+        ["tag1", time, {"a"=>2}]
+      ]
+
+      # create compressed entries
+      entries = ''
+      events.each do |_tag, _time, record|
+        v = [_time, record].to_msgpack
+        entries << compress(v)
+      end
+      chunk = ["tag1", entries, { 'compressed' => 'gzip' }].to_msgpack
+
+      # check CompressedMessagePackEventStream is created
+      mock(Fluent::CompressedMessagePackEventStream).new(entries, nil, 0)
+
+      d.run do
+        Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
+          option = d.instance.send(:on_message, obj, chunk.size, PEERADDR)
+          assert_equal 'gzip', option['compressed']
+        end
+      end
+      d.emits
+    end
+  end
+
   class Warning < self
     extend Fluent::Test::StartupShutdown
 
@@ -688,7 +752,7 @@ class ForwardInputTest < Test::Unit::TestCase
              auth: true
            }
          })
-    def test_packed_forward
+    def test_packed_forward(data)
       config = data[:config]
       options = data[:options]
       d = create_driver(config)
@@ -783,7 +847,7 @@ class ForwardInputTest < Test::Unit::TestCase
              auth: true
            }
          })
-    def test_message
+    def test_message(data)
       config = data[:config]
       options = data[:options]
       d = create_driver(config)
@@ -805,7 +869,7 @@ class ForwardInputTest < Test::Unit::TestCase
       end
 
       assert_equal events, d.emits
-      assert_equal ["", ""], @responses
+      assert_equal [nil, nil], @responses
 
       sleep 0.1 while d.instance.instance_eval{ @thread }.status # to confirm that plugin stopped completely
     end