fluentd 0.14.6 → 0.14.7

data/test/plugin/test_output_as_buffered.rb

@@ -56,6 +56,47 @@ module FluentPluginOutputAsBufferedTest
       super
     end
   end
+  class DummyStandardBufferedOutput < DummyBareOutput
+    def initialize
+      super
+      @prefer_delayed_commit = nil
+      @write = nil
+      @try_write = nil
+    end
+    def prefer_delayed_commit
+      @prefer_delayed_commit ? @prefer_delayed_commit.call : false
+    end
+    def write(chunk)
+      @write ? @write.call(chunk) : nil
+    end
+    def try_write(chunk)
+      @try_write ? @try_write.call(chunk) : nil
+    end
+  end
+  class DummyCustomFormatBufferedOutput < DummyBareOutput
+    def initialize
+      super
+      @format_type_is_msgpack = nil
+      @prefer_delayed_commit = nil
+      @write = nil
+      @try_write = nil
+    end
+    def format(tag, time, record)
+      @format ? @format.call(tag, time, record) : [tag, time, record].to_json
+    end
+    def formatted_to_msgpack_binary
+      @format_type_is_msgpack ? @format_type_is_msgpack.call : false
+    end
+    def prefer_delayed_commit
+      @prefer_delayed_commit ? @prefer_delayed_commit.call : false
+    end
+    def write(chunk)
+      @write ? @write.call(chunk) : nil
+    end
+    def try_write(chunk)
+      @try_write ? @try_write.call(chunk) : nil
+    end
+  end
   class DummyFullFeatureOutput < DummyBareOutput
     def initialize
       super
@@ -94,6 +135,8 @@ class BufferedOutputTest < Test::Unit::TestCase
     when :sync     then FluentPluginOutputAsBufferedTest::DummySyncOutput.new
     when :buffered then FluentPluginOutputAsBufferedTest::DummyAsyncOutput.new
     when :delayed  then FluentPluginOutputAsBufferedTest::DummyDelayedOutput.new
+    when :standard then FluentPluginOutputAsBufferedTest::DummyStandardBufferedOutput.new
+    when :custom   then FluentPluginOutputAsBufferedTest::DummyCustomFormatBufferedOutput.new
     when :full     then FluentPluginOutputAsBufferedTest::DummyFullFeatureOutput.new
     else
       raise ArgumentError, "unknown type: #{type}"
@@ -125,6 +168,186 @@ class BufferedOutputTest < Test::Unit::TestCase
     Timecop.return
   end
 
+  sub_test_case 'chunk feature in #write for output plugins' do
+    setup do
+      @stored_global_logger = $log
+      $log = Fluent::Test::TestLogger.new
+      @hash = {
+        'flush_mode' => 'immediate',
+        'flush_thread_interval' => '0.01',
+        'flush_thread_burst_interval' => '0.01',
+      }
+    end
+
+    teardown do
+      $log = @stored_global_logger
+    end
+
+    test 'plugin using standard format can iterate chunk for time, record in #write' do
+      events_from_chunk = []
+      @i = create_output(:standard)
+      @i.configure(config_element('ROOT','',{},[config_element('buffer','',@hash)]))
+      @i.register(:prefer_delayed_commit){ false }
+      @i.register(:write){ |chunk| e = []; assert chunk.respond_to?(:each); chunk.each{|t,r| e << [t,r]}; events_from_chunk << [:write, e] }
+      @i.register(:try_write){ |chunk| e = []; assert chunk.respond_to?(:each); chunk.each{|t,r| e << [t,r]}; events_from_chunk << [:try_write, e] }
+      @i.start
+      @i.after_start
+
+      events = [
+        [event_time('2016-10-05 16:16:16 -0700'), {"message" => "yaaaaaaaaay!"}],
+        [event_time('2016-10-05 16:16:17 -0700'), {"message" => "yoooooooooy!"}],
+      ]
+
+      @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
+      @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
+
+      waiting(5){ sleep 0.1 until events_from_chunk.size == 2 }
+
+      assert_equal 2, events_from_chunk.size
+      2.times.each do |i|
+        assert_equal :write, events_from_chunk[i][0]
+        assert_equal events, events_from_chunk[i][1]
+      end
+    end
+
+    test 'plugin using standard format can iterate chunk for time, record in #try_write' do
+      events_from_chunk = []
+      @i = create_output(:standard)
+      @i.configure(config_element('ROOT','',{},[config_element('buffer','',@hash)]))
+      @i.register(:prefer_delayed_commit){ true }
+      @i.register(:write){ |chunk| e = []; assert chunk.respond_to?(:each); chunk.each{|t,r| e << [t,r]}; events_from_chunk << [:write, e] }
+      @i.register(:try_write){ |chunk| e = []; assert chunk.respond_to?(:each); chunk.each{|t,r| e << [t,r]}; events_from_chunk << [:try_write, e] }
+      @i.start
+      @i.after_start
+
+      events = [
+        [event_time('2016-10-05 16:16:16 -0700'), {"message" => "yaaaaaaaaay!"}],
+        [event_time('2016-10-05 16:16:17 -0700'), {"message" => "yoooooooooy!"}],
+      ]
+
+      @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
+      @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
+
+      waiting(5){ sleep 0.1 until events_from_chunk.size == 2 }
+
+      assert_equal 2, events_from_chunk.size
+      2.times.each do |i|
+        assert_equal :try_write, events_from_chunk[i][0]
+        assert_equal events, events_from_chunk[i][1]
+      end
+    end
+
+    test 'plugin using custom format cannot iterate chunk in #write' do
+      events_from_chunk = []
+      @i = create_output(:custom)
+      @i.configure(config_element('ROOT','',{},[config_element('buffer','',@hash)]))
+      @i.register(:prefer_delayed_commit){ false }
+      @i.register(:format){ |tag, time, record| [tag,time,record].to_json }
+      @i.register(:format_type_is_msgpack){ false }
+      @i.register(:write){ |chunk| assert !(chunk.respond_to?(:each)) }
+      @i.register(:try_write){ |chunk| assert !(chunk.respond_to?(:each)) }
+      @i.start
+      @i.after_start
+
+      events = [
+        [event_time('2016-10-05 16:16:16 -0700'), {"message" => "yaaaaaaaaay!"}],
+        [event_time('2016-10-05 16:16:17 -0700'), {"message" => "yoooooooooy!"}],
+      ]
+
+      @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
+
+      assert_equal 0, events_from_chunk.size
+    end
+
+    test 'plugin using custom format cannot iterate chunk in #try_write' do
+      events_from_chunk = []
+      @i = create_output(:custom)
+      @i.configure(config_element('ROOT','',{},[config_element('buffer','',@hash)]))
+      @i.register(:prefer_delayed_commit){ true }
+      @i.register(:format){ |tag, time, record| [tag,time,record].to_json }
+      @i.register(:format_type_is_msgpack){ false }
+      @i.register(:write){ |chunk| assert !(chunk.respond_to?(:each)) }
+      @i.register(:try_write){ |chunk| assert !(chunk.respond_to?(:each)) }
+      @i.start
+      @i.after_start
+
+      events = [
+        [event_time('2016-10-05 16:16:16 -0700'), {"message" => "yaaaaaaaaay!"}],
+        [event_time('2016-10-05 16:16:17 -0700'), {"message" => "yoooooooooy!"}],
+      ]
+
+      @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
+
+      assert_equal 0, events_from_chunk.size
+    end
+
+    test 'plugin using custom format can iterate chunk in #write if #format returns msgpack' do
+      events_from_chunk = []
+      @i = create_output(:custom)
+      @i.configure(config_element('ROOT','',{},[config_element('buffer','',@hash)]))
+      @i.register(:prefer_delayed_commit){ false }
+      @i.register(:format){ |tag, time, record| [tag,time,record].to_msgpack }
+      @i.register(:format_type_is_msgpack){ true }
+      @i.register(:write){ |chunk| e = []; assert chunk.respond_to?(:each); chunk.each{|ta,t,r| e << [ta,t,r]}; events_from_chunk << [:write, e] }
+      @i.register(:try_write){ |chunk| e = []; assert chunk.respond_to?(:each); chunk.each{|ta,t,r| e << [ta,t,r]}; events_from_chunk << [:try_write, e] }
+      @i.start
+      @i.after_start
+
+      events = [
+        [event_time('2016-10-05 16:16:16 -0700'), {"message" => "yaaaaaaaaay!"}],
+        [event_time('2016-10-05 16:16:17 -0700'), {"message" => "yoooooooooy!"}],
+      ]
+
+      @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
+      @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
+
+      waiting(5){ sleep 0.1 until events_from_chunk.size == 2 }
+
+      assert_equal 2, events_from_chunk.size
+      2.times.each do |i|
+        assert_equal :write, events_from_chunk[i][0]
+        each_pushed = events_from_chunk[i][1]
+        assert_equal 2, each_pushed.size
+        assert_equal 'test.tag', each_pushed[0][0]
+        assert_equal 'test.tag', each_pushed[1][0]
+        assert_equal events, each_pushed.map{|tag,time,record| [time,record]}
+      end
+    end
+
+    test 'plugin using custom format can iterate chunk in #try_write if #format returns msgpack' do
+      events_from_chunk = []
+      @i = create_output(:custom)
+      @i.configure(config_element('ROOT','',{},[config_element('buffer','',@hash)]))
+      @i.register(:prefer_delayed_commit){ true }
+      @i.register(:format){ |tag, time, record| [tag,time,record].to_msgpack }
+      @i.register(:format_type_is_msgpack){ true }
+      @i.register(:write){ |chunk| events_from_chunk = []; assert chunk.respond_to?(:each); chunk.each{|ta,t,r| e << [ta,t,r]}; events_from_chunk << [:write, e] }
+      @i.register(:try_write){ |chunk| e = []; assert chunk.respond_to?(:each); chunk.each{|ta,t,r| e << [ta,t,r]}; events_from_chunk << [:try_write, e] }
+      @i.start
+      @i.after_start
+
+      events = [
+        [event_time('2016-10-05 16:16:16 -0700'), {"message" => "yaaaaaaaaay!"}],
+        [event_time('2016-10-05 16:16:17 -0700'), {"message" => "yoooooooooy!"}],
+      ]
+
+      @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
+      @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
+
+      waiting(5){ sleep 0.1 until events_from_chunk.size == 2 }
+
+      assert_equal 2, events_from_chunk.size
+      2.times.each do |i|
+        assert_equal :try_write, events_from_chunk[i][0]
+        each_pushed = events_from_chunk[i][1]
+        assert_equal 2, each_pushed.size
+        assert_equal 'test.tag', each_pushed[0][0]
+        assert_equal 'test.tag', each_pushed[1][0]
+        assert_equal events, each_pushed.map{|tag,time,record| [time,record]}
+      end
+    end
+  end
+
   sub_test_case 'buffered output configured with many chunk keys' do
     setup do
       @stored_global_logger = $log

data/test/plugin/test_output_as_buffered_compress.rb

@@ -0,0 +1,165 @@
+require_relative '../helper'
+require 'fluent/plugin/output'
+require 'fluent/plugin/buffer'
+require 'fluent/plugin/compressable'
+require 'fluent/event'
+
+require 'timeout'
+
+module FluentPluginOutputAsBufferedCompressTest
+  class DummyBareOutput < Fluent::Plugin::Output
+    def register(name, &block)
+      instance_variable_set("@#{name}", block)
+    end
+  end
+
+  class DummyAsyncOutput < DummyBareOutput
+    def initialize
+      super
+      @format = @write = nil
+    end
+    def write(chunk)
+      @write ? @write.call(chunk) : nil
+    end
+  end
+
+  class DummyAsyncOutputWithFormat < DummyBareOutput
+    def initialize
+      super
+      @format = nil
+    end
+    def write(chunk)
+      @write ? @write.call(chunk) : nil
+    end
+    def format(tag, time, record)
+      @format ? @format.call(tag, time, record) : [tag, time, record].to_json
+    end
+  end
+end
+
+class BufferedOutputCompressTest < Test::Unit::TestCase
+  include Fluent::Plugin::Compressable
+
+  def create_output(type=:async)
+    case type
+    when :async then FluentPluginOutputAsBufferedCompressTest::DummyAsyncOutput.new
+    when :async_with_format then FluentPluginOutputAsBufferedCompressTest::DummyAsyncOutputWithFormat.new
+    else
+      raise ArgumentError, "unknown type: #{type}"
+    end
+  end
+
+  def waiting(seconds)
+    begin
+      Timeout.timeout(seconds) do
+        yield
+      end
+    rescue Timeout::Error
+      STDERR.print(*@i.log.out.logs)
+      raise
+    end
+  end
+
+  def dummy_event_stream
+    Fluent::ArrayEventStream.new(
+      [
+        [event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
+        [event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
+        [event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
+      ]
+    )
+  end
+
+  TMP_DIR = File.expand_path('../../tmp/test_output_as_buffered_compress', __FILE__)
+
+  setup do
+    FileUtils.rm_r TMP_DIR rescue nil
+    FileUtils.mkdir_p TMP_DIR
+  end
+
+  teardown do
+    if @i
+      @i.stop unless @i.stopped?
+      @i.before_shutdown unless @i.before_shutdown?
+      @i.shutdown unless @i.shutdown?
+      @i.after_shutdown unless @i.after_shutdown?
+      @i.close unless @i.closed?
+      @i.terminate unless @i.terminated?
+    end
+  end
+
+  data(
+    handle_simple_stream: config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
+    handle_stream_with_standard_format:  config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
+    handle_simple_stream_and_file_chunk: config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
+    handle_stream_with_standard_format_and_file_chunk:  config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
+  )
+  test 'call a standard format when output plugin adds data to chunk' do |buffer_config|
+    @i = create_output(:async)
+    @i.configure(config_element('ROOT','', {}, [buffer_config]))
+    @i.start
+    @i.after_start
+
+    io = StringIO.new
+    es = dummy_event_stream
+    expected = es.map { |e| e }
+    compressed_data = ''
+
+    assert_equal :gzip, @i.buffer.compress
+
+    @i.register(:write) do |c|
+      compressed_data = c.instance_variable_get(:@chunk)
+      if compressed_data.is_a?(File)
+        compressed_data.seek(0, IO::SEEK_SET)
+        compressed_data = compressed_data.read
+      end
+      c.write_to(io)
+    end
+
+    @i.emit_events('tag', es)
+    @i.enqueue_thread_wait
+    @i.flush_thread_wakeup
+    waiting(4) { Thread.pass until io.size > 0 }
+
+    assert_equal expected, Fluent::MessagePackEventStream.new(decompress(compressed_data)).map { |t, r| [t, r] }
+    assert_equal expected, Fluent::MessagePackEventStream.new(io.string).map { |t, r| [t, r] }
+  end
+
+  data(
+    handle_simple_stream: config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
+    handle_stream_with_custom_format:  config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
+    handle_simple_stream_and_file_chunk: config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
+    handle_stream_with_custom_format_and_file_chunk:  config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
+  )
+  test 'call a custom format when output plugin adds data to chunk' do |buffer_config|
+    @i = create_output(:async_with_format)
+    @i.configure(config_element('ROOT','', {}, [buffer_config]))
+    @i.start
+    @i.after_start
+
+    io = StringIO.new
+    es = dummy_event_stream
+    expected = es.map { |e| "#{e[1]}\n" }.join # e[1] is record
+    compressed_data = ''
+
+    assert_equal :gzip, @i.buffer.compress
+
+    @i.register(:format) { |tag, time, record| "#{record}\n" }
+    @i.register(:write) { |c|
+      compressed_data = c.instance_variable_get(:@chunk)
+      if compressed_data.is_a?(File)
+        compressed_data.seek(0, IO::SEEK_SET)
+        compressed_data = compressed_data.read
+      end
+      c.write_to(io)
+    }
+
+    @i.emit_events('tag', es)
+    @i.enqueue_thread_wait
+    @i.flush_thread_wakeup
+    waiting(4) { Thread.pass until io.size > 0 }
+
+    assert_equal expected, decompress(compressed_data)
+    assert_equal expected, io.string
+  end
+end

data/test/plugin/test_parser_json.rb

@@ -56,6 +56,14 @@ class JsonParserTest < ::Test::Unit::TestCase
     }
   end
 
+  data('oj' => 'oj', 'yajl' => 'yajl')
+  def test_parse_with_colon_string(data)
+    @parser.configure('json_parser' => data)
+    @parser.instance.parse('{"time":1362020400,"log":":message"}') { |time, record|
+      assert_equal(record['log'], ':message')
+    }
+  end
+
   data('oj' => 'oj', 'yajl' => 'yajl')
   def test_parse_with_invalid_time(data)
     @parser.configure('json_parser' => data)

data/test/plugin/test_parser_regexp.rb

@@ -44,7 +44,7 @@ class RegexpParserTest < ::Test::Unit::TestCase
     end
 
     def test_parse_with_configure
-      # Specify conf by configure method instaed of intializer
+      # Specify conf by configure method instead of initializer
       regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
       conf = {
         'time_format' => "%d/%b/%Y:%H:%M:%S %z",

data/test/plugin_helper/test_child_process.rb

@@ -213,7 +213,7 @@ class ChildProcessTest < Test::Unit::TestCase
       end
       sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
       sleep TEST_WAIT_INTERVAL_FOR_LOOP * 10
-      @d.stop # nothing occures
+      @d.stop # nothing occurs
       @d.shutdown
 
       assert{ ary.size > 5 }
@@ -249,7 +249,7 @@ class ChildProcessTest < Test::Unit::TestCase
 
         assert_equal [], @d.log.out.logs
 
-        @d.stop # nothing occures
+        @d.stop # nothing occurs
         sleep TEST_WAIT_INTERVAL_FOR_LOOP * 5
         lines1 = ary.size
         assert{ lines1 > 1 }