fluentd 0.14.1 → 0.14.2

data/test/plugin/test_buffer_chunk.rb

@@ -10,7 +10,8 @@ class BufferChunkTest < Test::Unit::TestCase
       assert{ chunk.metadata.object_id == meta.object_id }
       assert{ chunk.created_at.is_a? Time }
       assert{ chunk.modified_at.is_a? Time }
-      assert chunk.staged?
+      assert chunk.unstaged?
+      assert !chunk.staged?
       assert !chunk.queued?
       assert !chunk.closed?
     end
@@ -20,6 +21,7 @@ class BufferChunkTest < Test::Unit::TestCase
       chunk = Fluent::Plugin::Buffer::Chunk.new(meta)
 
       assert chunk.respond_to?(:append)
+      assert chunk.respond_to?(:concat)
       assert chunk.respond_to?(:commit)
       assert chunk.respond_to?(:rollback)
       assert chunk.respond_to?(:bytesize)
@@ -30,7 +32,8 @@ class BufferChunkTest < Test::Unit::TestCase
       assert chunk.respond_to?(:open)
       assert chunk.respond_to?(:write_to)
       assert chunk.respond_to?(:msgpack_each)
-      assert_raise(NotImplementedError){ chunk.append(nil) }
+      assert_raise(NotImplementedError){ chunk.append([]) }
+      assert_raise(NotImplementedError){ chunk.concat(nil, 0) }
       assert_raise(NotImplementedError){ chunk.commit }
       assert_raise(NotImplementedError){ chunk.rollback }
       assert_raise(NotImplementedError){ chunk.bytesize }
@@ -63,21 +66,76 @@ class BufferChunkTest < Test::Unit::TestCase
   sub_test_case 'minimum chunk implements #size and #open' do
     test 'chunk lifecycle' do
       c = TestChunk.new(Object.new)
+      assert c.unstaged?
+      assert !c.staged?
+      assert !c.queued?
+      assert !c.closed?
+      assert c.writable?
+
+      c.staged!
+
+      assert !c.unstaged?
       assert c.staged?
       assert !c.queued?
       assert !c.closed?
+      assert c.writable?
+
+      c.enqueued!
+
+      assert !c.unstaged?
+      assert !c.staged?
+      assert c.queued?
+      assert !c.closed?
+      assert !c.writable?
+
+      c.close
+
+      assert !c.unstaged?
+      assert !c.staged?
+      assert !c.queued?
+      assert c.closed?
+      assert !c.writable?
+    end
+
+    test 'chunk can be unstaged' do
+      c = TestChunk.new(Object.new)
+      assert c.unstaged?
+      assert !c.staged?
+      assert !c.queued?
+      assert !c.closed?
+      assert c.writable?
+
+      c.staged!
+
+      assert !c.unstaged?
+      assert c.staged?
+      assert !c.queued?
+      assert !c.closed?
+      assert c.writable?
+
+      c.unstaged!
+
+      assert c.unstaged?
+      assert !c.staged?
+      assert !c.queued?
+      assert !c.closed?
+      assert c.writable?
 
       c.enqueued!
 
+      assert !c.unstaged?
       assert !c.staged?
       assert c.queued?
       assert !c.closed?
+      assert !c.writable?
 
       c.close
 
+      assert !c.unstaged?
       assert !c.staged?
       assert !c.queued?
       assert c.closed?
+      assert !c.writable?
     end
 
     test 'can respond to #empty? correctly' do

data/test/plugin/test_buffer_file_chunk.rb

@@ -49,8 +49,9 @@ class BufferFileChunkTest < Test::Unit::TestCase
     data(
       correct_staged: ['/mydir/mypath/myfile.b00ff.log', :staged],
       correct_queued: ['/mydir/mypath/myfile.q00ff.log', :queued],
-      incorrect_staged: ['/mydir/mypath/myfile.b00ff.log/unknown', :queued],
-      incorrect_queued: ['/mydir/mypath/myfile.q00ff.log/unknown', :queued],
+      incorrect_staged: ['/mydir/mypath/myfile.b00ff.log/unknown', :unknown],
+      incorrect_queued: ['/mydir/mypath/myfile.q00ff.log/unknown', :unknown],
+      output_file: ['/mydir/mypath/myfile.20160716.log', :unknown],
     )
     test 'can .assume_chunk_state' do |data|
       path, expected = data
@@ -122,7 +123,7 @@ class BufferFileChunkTest < Test::Unit::TestCase
       assert File.exist?(gen_chunk_path('b', @c.unique_id) + '.meta')
       assert{ File.stat(gen_chunk_path('b', @c.unique_id) + '.meta').mode.to_s(8).end_with?(@klass.const_get('FILE_PERMISSION').to_s(8)) }
 
-      assert_equal :staged, @c.state
+      assert_equal :unstaged, @c.state
       assert @c.empty?
     end
 

data/test/plugin/test_filter_grep.rb

@@ -1,16 +1,17 @@
 require_relative '../helper'
 require 'fluent/plugin/filter_grep'
+require 'fluent/test/driver/filter'
 
 class GrepFilterTest < Test::Unit::TestCase
   include Fluent
 
   setup do
     Fluent::Test.setup
-    @time = Fluent::Engine.now
+    @time = event_time
   end
 
   def create_driver(conf = '')
-    Test::FilterTestDriver.new(GrepFilter).configure(conf, true)
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::GrepFilter).configure(conf)
   end
 
   sub_test_case 'configure' do
@@ -41,35 +42,36 @@ class GrepFilterTest < Test::Unit::TestCase
       ]
     end
 
-    def emit(config, msgs)
+    def filter(config, msgs)
       d = create_driver(config)
       d.run {
         msgs.each { |msg|
-          d.emit({'foo' => 'bar', 'message' => msg}, @time)
+          d.feed("filter.test", @time, {'foo' => 'bar', 'message' => msg})
         }
-      }.filtered
+      }
+      d.filtered_records
     end
 
     test 'empty config' do
-      es = emit('', messages)
-      assert_equal(4, es.instance_variable_get(:@record_array).size)
+      filtered_records = filter('', messages)
+      assert_equal(4, filtered_records.size)
     end
 
     test 'regexpN' do
-      es = emit('regexp1 message WARN', messages)
-      assert_equal(3, es.instance_variable_get(:@record_array).size)
+      filtered_records = filter('regexp1 message WARN', messages)
+      assert_equal(3, filtered_records.size)
       assert_block('only WARN logs') do
-        es.all? { |t, r|
+        filtered_records.all? { |r|
           !r['message'].include?('INFO')
         }
       end
     end
 
     test 'excludeN' do
-      es = emit('exclude1 message favicon', messages)
-      assert_equal(3, es.instance_variable_get(:@record_array).size)
+      filtered_records = filter('exclude1 message favicon', messages)
+      assert_equal(3, filtered_records.size)
       assert_block('remove favicon logs') do
-        es.all? { |t, r|
+        filtered_records.all? { |r|
           !r['message'].include?('favicon')
         }
       end
@@ -84,17 +86,19 @@ class GrepFilterTest < Test::Unit::TestCase
 
       test "don't raise an exception" do
         assert_nothing_raised { 
-          emit(%[regexp1 message WARN], ["\xff".force_encoding('UTF-8')])
+          filter(%[regexp1 message WARN], ["\xff".force_encoding('UTF-8')])
         }
       end
     end
   end
 
   sub_test_case 'grep non-string jsonable values' do
-    def emit(msg, config = 'regexp1 message 0')
+    def filter(msg, config = 'regexp1 message 0')
       d = create_driver(config)
-      d.emit({'foo' => 'bar', 'message' => msg}, @time)
-      d.run.filtered
+      d.run do
+        d.feed("filter.test", @time, {'foo' => 'bar', 'message' => msg})
+      end
+      d.filtered_records
     end
 
     data(
@@ -103,13 +107,13 @@ class GrepFilterTest < Test::Unit::TestCase
       'integer' => 0,
       'float' => 0.1)
     test "value" do |data|
-      es = emit(data)
-      assert_equal(1, es.instance_variable_get(:@record_array).size)
+      filtered_records = filter(data)
+      assert_equal(1, filtered_records.size)
     end
 
     test "value boolean" do
-      es = emit(true, %[regexp1 message true])
-      assert_equal(1, es.instance_variable_get(:@record_array).size)
+      filtered_records = filter(true, %[regexp1 message true])
+      assert_equal(1, filtered_records.size)
     end
   end
 end

data/test/plugin/test_filter_record_transformer.rb

@@ -1,5 +1,6 @@
 require_relative '../helper'
 require 'timecop'
+require 'fluent/test/driver/filter'
 require 'fluent/plugin/filter_record_transformer'
 
 class RecordTransformerFilterTest < Test::Unit::TestCase
@@ -10,7 +11,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
     @hostname = Socket.gethostname.chomp
     @tag = 'test.tag'
     @tag_parts = @tag.split('.')
-    @time = Time.utc(1,2,3,4,5,2010,nil,nil,nil,nil)
+    @time = event_time('2010-05-04 03:02:01 UTC')
     Timecop.freeze(@time)
   end
 
@@ -19,7 +20,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
   end
 
   def create_driver(conf = '')
-    Test::FilterTestDriver.new(RecordTransformerFilter, @tag).configure(conf, true)
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::RecordTransformerFilter).configure(conf)
   end
 
   sub_test_case 'configure' do
@@ -37,13 +38,14 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
   end
 
   sub_test_case "test options" do
-    def emit(config, msgs = [''])
+    def filter(config, msgs = [''])
       d = create_driver(config)
       d.run {
         msgs.each { |msg|
-          d.emit({'foo' => 'bar', 'message' => msg}, @time)
+          d.feed(@tag, @time, {'foo' => 'bar', 'message' => msg})
         }
-      }.filtered
+      }
+      d.filtered
     end
 
     CONFIG = %[
@@ -57,24 +59,24 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
 
     test 'typical usage' do
       msgs = ['1', '2']
-      es = emit(CONFIG, msgs)
-      es.each_with_index do |(_t, r), i|
+      filtered = filter(CONFIG, msgs)
+      filtered.each_with_index do |(_t, r), i|
         assert_equal('bar', r['foo'])
         assert_equal(@hostname, r['hostname'])
         assert_equal(@tag, r['tag'])
-        assert_equal(@time.to_s, r['time'])
+        assert_equal(Time.at(@time).localtime.to_s, r['time'])
         assert_equal("#{@hostname} #{@tag_parts[-1]} #{msgs[i]}", r['message'])
       end
     end
 
     test 'remove_keys' do
       config = CONFIG + %[remove_keys foo,message]
-      es = emit(config)
-      es.each_with_index do |(_t, r), i|
+      filtered = filter(config)
+      filtered.each_with_index do |(_t, r), i|
         assert_not_include(r, 'foo')
         assert_equal(@hostname, r['hostname'])
         assert_equal(@tag, r['tag'])
-        assert_equal(@time.to_s, r['time'])
+        assert_equal(Time.at(@time).localtime.to_s, r['time'])
         assert_not_include(r, 'message')
       end
     end
@@ -82,12 +84,12 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
     test 'renew_record' do
       config = CONFIG + %[renew_record true]
       msgs = ['1', '2']
-      es = emit(config, msgs)
-      es.each_with_index do |(_t, r), i|
+      filtered = filter(config, msgs)
+      filtered.each_with_index do |(_t, r), i|
         assert_not_include(r, 'foo')
         assert_equal(@hostname, r['hostname'])
         assert_equal(@tag, r['tag'])
-        assert_equal(@time.to_s, r['time'])
+        assert_equal(Time.at(@time).localtime.to_s, r['time'])
         assert_equal("#{@hostname} #{@tag_parts[-1]} #{msgs[i]}", r['message'])
       end
     end
@@ -96,8 +98,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       config = %[renew_time_key message]
       times = [ Time.local(2,2,3,4,5,2010,nil,nil,nil,nil), Time.local(3,2,3,4,5,2010,nil,nil,nil,nil) ]
       msgs = times.map{|t| t.to_f.to_s }
-      es = emit(config, msgs)
-      es.each_with_index do |(time, _record), i|
+      filtered = filter(config, msgs)
+      filtered.each_with_index do |(time, _record), i|
         assert_equal(times[i].to_i, time)
         assert(time.is_a?(Fluent::EventTime))
       end
@@ -106,8 +108,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
     test 'keep_keys' do
       config = %[renew_record true\nkeep_keys foo,message]
       msgs = ['1', '2']
-      es = emit(config, msgs)
-      es.each_with_index do |(_t, r), i|
+      filtered = filter(config, msgs)
+      filtered.each_with_index do |(_t, r), i|
         assert_equal('bar', r['foo'])
         assert_equal(msgs[i], r['message'])
       end
@@ -121,8 +123,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         </record>
       ]
       msgs = ['1', '2']
-      es = emit(config, msgs)
-      es.each_with_index do |(_t, r), i|
+      filtered = filter(config, msgs)
+      filtered.each_with_index do |(_t, r), i|
         assert_equal("#{@hostname} #{@tag_parts[-1]} '#{msgs[i]}'", r['message'])
       end
     end
@@ -134,8 +136,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         </record>
       %]
       msgs = ['1', '2']
-      es = emit(config, msgs)
-      es.each_with_index do |(_t, r), i|
+      filtered = filter(config, msgs)
+      filtered.each_with_index do |(_t, r), i|
         assert_equal({"k1"=>100, "k2"=>"foobar"}, r['hash_field'])
       end
     end
@@ -147,8 +149,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         </record>
       %]
       msgs = ['1', '2']
-      es = emit(config, msgs)
-      es.each_with_index do |(_t, r), i|
+      filtered = filter(config, msgs)
+      filtered.each_with_index do |(_t, r), i|
         assert_equal([1,2,3], r['array_field'])
       end
     end
@@ -160,15 +162,15 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         </record>
       %]
       msgs = ['1', '2']
-      es = emit(config, msgs)
-      es.each_with_index do |(_t, r), i|
+      filtered = filter(config, msgs)
+      filtered.each_with_index do |(_t, r), i|
         assert_equal({"hello"=>[1,2,3], "world"=>{"foo"=>"bar"}}, r['mixed_field'])
       end
     end
   end
 
   sub_test_case 'test placeholders' do
-    def emit(config, msgs = [''])
+    def filter(config, msgs = [''])
       d = create_driver(config)
       yield d if block_given?
       d.run {
@@ -178,9 +180,10 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
             'message'    => msg,
           }
           record = record.merge(msg) if msg.is_a?(Hash)
-          d.emit(record, @time)
+          d.feed(@tag, @time, record)
         end
-      }.filtered
+      }
+      d.filtered
     end
 
     %w[yes no].each do |enable_ruby|
@@ -191,8 +194,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
             message ${hostname}
           </record>
         ]
-        es = emit(config)
-        es.each do |t, r|
+        filtered = filter(config)
+        filtered.each do |t, r|
           assert_equal(@hostname, r['message'])
         end
       end
@@ -204,8 +207,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
             message ${tag}
           </record>
         ]
-        es = emit(config)
-        es.each do |t, r|
+        filtered = filter(config)
+        filtered.each do |t, r|
           assert_equal(@tag, r['message'])
         end
       end
@@ -218,8 +221,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           </record>
         ]
         expected = "#{@tag.split('.').first} #{@tag.split('.').last}"
-        es = emit(config)
-        es.each do |t, r|
+        filtered = filter(config)
+        filtered.each do |t, r|
           assert_equal(expected, r['message'])
         end
       end
@@ -233,8 +236,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         ]
         @tag = 'prefix.test.tag.suffix'
         expected = "prefix.test prefix.test.tag tag.suffix test.tag.suffix"
-        es = emit(config)
-        es.each do |t, r|
+        filtered = filter(config)
+        filtered.each do |t, r|
           assert_equal(expected, r['message'])
         end
       end
@@ -246,9 +249,9 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
             message ${time}
           </record>
         ]
-        es = emit(config)
-        es.each do |t, r|
-          assert_equal(@time.to_s, r['message'])
+        filtered = filter(config)
+        filtered.each do |t, r|
+          assert_equal(Time.at(@time).localtime.to_s, r['message'])
         end
       end
 
@@ -262,8 +265,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           </record>
         ]
         msgs = ['1', '2']
-        es = emit(config, msgs)
-        es.each_with_index do |(_t, r), i|
+        filtered = filter(config, msgs)
+        filtered.each_with_index do |(_t, r), i|
           assert_not_include(r, 'eventType0')
           assert_equal("bar", r['eventtype'])
           assert_equal("bar #{msgs[i]}", r['message'])
@@ -282,8 +285,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           </record>
         ]
         msgs = ['1', '2']
-        es = emit(config, msgs)
-        es.each_with_index do |(_t, r), i|
+        filtered = filter(config, msgs)
+        filtered.each_with_index do |(_t, r), i|
           assert_equal({"hostname" => @hostname, "tag" => @tag, "#{@tag}" => 100}, r['hash_field'])
         end
       end
@@ -296,8 +299,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           </record>
         ]
         msgs = ['1', '2']
-        es = emit(config, msgs)
-        es.each_with_index do |(_t, r), i|
+        filtered = filter(config, msgs)
+        filtered.each_with_index do |(_t, r), i|
           assert_equal([@hostname, @tag], r['array_field'])
         end
       end
@@ -310,8 +313,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           </record>
         ]
         msgs = ['1', '2']
-        es = emit(config, msgs)
-        es.each_with_index do |(_t, r), i|
+        filtered = filter(config, msgs)
+        filtered.each_with_index do |(_t, r), i|
           assert_equal([{"tag" => @tag}], r['mixed_field'])
         end
       end
@@ -326,8 +329,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           </record>
         ]
         msgs = ['1', '2']
-        es = emit(config, msgs)
-        es.each_with_index do |(_t, r), i|
+        filtered = filter(config, msgs)
+        filtered.each_with_index do |(_t, r), i|
           assert_equal({@hostname=>'hostname',"foo.#{@tag}"=>'tag'}, r)
         end
       end
@@ -379,8 +382,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
             with_quote: %Q{source[""]} },
         ]
         actual_results = []
-        es = emit(config, msgs)
-        es.each_with_index do |(_t, r), i|
+        filtered = filter(config, msgs)
+        filtered.each_with_index do |(_t, r), i|
           actual_results << {
             single: r["single"],
             multiple: r["multiple"],
@@ -433,8 +436,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
             with_suffix: "#{nil.to_s}-suffix" },
         ]
         actual_results = []
-        es = emit(config, msgs)
-        es.each_with_index do |(_t, r), i|
+        filtered = filter(config, msgs)
+        filtered.each_with_index do |(_t, r), i|
           actual_results << {
             single: r["single"],
             multiple: r["multiple"],
@@ -459,8 +462,9 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           "foo.bar"    => "foo.bar",
           "@timestamp" => 10,
         }
-        es = d.run { d.emit(record, @time) }.filtered
-        es.each do |t, r|
+        d.run { d.feed(@tag, @time, record) }
+        filtered = d.filtered
+        filtered.each do |t, r|
           assert { r['_timestamp'] == record['@timestamp'] }
           assert { r['_foo_bar'] == record['foo.bar'] }
         end
@@ -474,7 +478,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           message ${unknown}
         </record>
       ]
-      emit(config) { |d|
+      filter(config) { |d|
         mock(d.instance.log).warn("unknown placeholder `${unknown}` found")
       }
     end
@@ -486,10 +490,10 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           message ${unknown['bar']}
         </record>
       ]
-      es = emit(config) { |d|
+      filtered = filter(config) { |d|
         mock(d.instance.log).warn("failed to expand `%Q[\#{unknown['bar']}]`", anything)
       }
-      es.each do |t, r|
+      filtered.each do |t, r|
         assert_nil(r['message'])
       end
     end
@@ -503,8 +507,9 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       ]
       d = create_driver(config)
       message = {"@timestamp" => "foo"}
-      es = d.run { d.emit(message, @time) }.filtered
-      es.each do |t, r|
+      d.run { d.feed(@tag, @time, message) }
+      filtered = d.filtered
+      filtered.each do |t, r|
         assert_equal(message["@timestamp"], r['foo'])
       end
     end
@@ -520,8 +525,9 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       ]
       d = create_driver(config)
       message = {"@timestamp" => "foo"}
-      es = d.run { d.emit(message, @time) }.filtered
-      es.each do |t, r|
+      d.run { d.feed(@tag, @time, message) }
+      filtered = d.filtered
+      filtered.each do |t, r|
         assert_equal([message["@timestamp"]], r['foo'])
       end
     end
@@ -535,8 +541,9 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       ]
       d = create_driver(config)
       message = {"@timestamp" => "foo"}
-      es = d.run { d.emit(message, @time) }.filtered
-      es.each do |t, r|
+      d.run { d.feed(@tag, @time, message) }
+      filtered = d.filtered
+      filtered.each do |t, r|
         assert_equal(message["@timestamp"], r['foo'])
       end
     end
@@ -563,8 +570,9 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       "@timestamp" => 10,
       "message"    => "10",
     }
-    es = d.run { d.emit(record, @time) }.filtered
-    es.each do |t, r|
+    d.run { d.feed(@tag, @time, record) }
+    filtered = d.filtered
+    filtered.each do |t, r|
       assert { r['_message'] == "prefix-#{record['message']}-suffix" }
       assert { r['_time'] == Time.at(@time) }
       assert { r['_number'] == 0 }