fluentd 0.14.1 → 0.14.2

data/test/plugin/test_output_as_buffered_secondary.rb

@@ -208,6 +208,8 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       @i.secondary.register(:write){|chunk| chunk.read.split("\n").each{|line| written << JSON.parse(line) } }
       @i.start
 
+      @i.interrupt_flushes
+
       now = Time.parse('2016-04-13 18:33:30 -0700')
       Timecop.freeze( now )
 
@@ -270,6 +272,8 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       @i.secondary.register(:write){|chunk| chunk.read.split("\n").each{|line| written << JSON.parse(line) } }
       @i.start
 
+      @i.interrupt_flushes
+
       now = Time.parse('2016-04-13 18:33:30 -0700')
       Timecop.freeze( now )
 
@@ -333,6 +337,8 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       @i.secondary.register(:try_write){|chunk| chunks << chunk; chunk.read.split("\n").each{|line| written << JSON.parse(line) } }
       @i.start
 
+      @i.interrupt_flushes
+
       now = Time.parse('2016-04-13 18:33:30 -0700')
       Timecop.freeze( now )
 
@@ -407,6 +413,8 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       @i.secondary.register(:try_write){|chunk| chunks << chunk; chunk.read.split("\n").each{|line| written << JSON.parse(line) } }
       @i.start
 
+      @i.interrupt_flushes
+
       now = Time.parse('2016-04-13 18:33:30 -0700')
       Timecop.freeze( now )
 
@@ -482,6 +490,8 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       @i.secondary.register(:write){|chunk| raise "don't use this" }
       @i.start
 
+      @i.interrupt_flushes
+
       now = Time.parse('2016-04-13 18:33:30 -0700')
       Timecop.freeze( now )
 
@@ -564,6 +574,8 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       @i.secondary.register(:write){|chunk| raise "your secondary is also useless." }
       @i.start
 
+      @i.interrupt_flushes
+
       now = Time.parse('2016-04-13 18:33:30 -0700')
       Timecop.freeze( now )
 
@@ -632,6 +644,8 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       @i.secondary.register(:write){|chunk| chunk.read.split("\n").each{|line| written << JSON.parse(line) } }
       @i.start
 
+      @i.interrupt_flushes
+
       now = Time.parse('2016-04-13 18:33:30 -0700')
       Timecop.freeze( now )
 
@@ -700,6 +714,8 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       @i.secondary.register(:write){|chunk| raise "your secondary is also useless." }
       @i.start
 
+      @i.interrupt_flushes
+
       now = Time.parse('2016-04-13 18:33:30 -0700')
       Timecop.freeze( now )
 

data/test/plugin/test_output_as_standard.rb

@@ -90,7 +90,7 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
       es = test_event_stream
 
       buffer_mock = flexmock(@i.buffer)
-      buffer_mock.should_receive(:write).once.with({m => [es.to_msgpack_stream, es.size]}, bulk: true, enqueue: false)
+      buffer_mock.should_receive(:write).once.with({m => es}, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, enqueue: false)
 
       @i.execute_chunking("mytag.test", es)
     end
@@ -104,7 +104,7 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
       es = test_event_stream
 
       buffer_mock = flexmock(@i.buffer)
-      buffer_mock.should_receive(:write).once.with({m => [es.to_msgpack_stream(time_int: true), es.size]}, bulk: true, enqueue: false)
+      buffer_mock.should_receive(:write).once.with({m => es}, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM_TIME_INT, enqueue: false)
 
       @i.execute_chunking("mytag.test", es)
     end
@@ -120,7 +120,7 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
       es = test_event_stream
 
       buffer_mock = flexmock(@i.buffer)
-      buffer_mock.should_receive(:write).once.with({m => [es.to_msgpack_stream, es.size]}, bulk: true, enqueue: false)
+      buffer_mock.should_receive(:write).once.with({m => es}, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, enqueue: false)
 
       @i.execute_chunking("mytag.test", es)
     end
@@ -134,7 +134,7 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
       es = test_event_stream
 
       buffer_mock = flexmock(@i.buffer)
-      buffer_mock.should_receive(:write).once.with({m => [es.to_msgpack_stream(time_int: true), es.size]}, bulk: true, enqueue: false)
+      buffer_mock.should_receive(:write).once.with({m => es}, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM_TIME_INT, enqueue: false)
 
       @i.execute_chunking("mytag.test", es)
     end
@@ -164,10 +164,10 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
 
       buffer_mock = flexmock(@i.buffer)
       buffer_mock.should_receive(:write).once.with({
-          m1 => [es1.to_msgpack_stream, 3],
-          m2 => [es2.to_msgpack_stream, 2],
-          m3 => [es3.to_msgpack_stream, 1],
-        }, bulk: true, enqueue: false)
+          m1 => es1,
+          m2 => es2,
+          m3 => es3,
+        }, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, enqueue: false)
 
       es = test_event_stream
       @i.execute_chunking("mytag.test", es)
@@ -196,10 +196,10 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
 
       buffer_mock = flexmock(@i.buffer)
       buffer_mock.should_receive(:write).with({
-          m1 => [es1.to_msgpack_stream(time_int: true), 3],
-          m2 => [es2.to_msgpack_stream(time_int: true), 2],
-          m3 => [es3.to_msgpack_stream(time_int: true), 1],
-        }, bulk: true, enqueue: false)
+          m1 => es1,
+          m2 => es2,
+          m3 => es3,
+        }, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM_TIME_INT, enqueue: false)
 
       es = test_event_stream
       @i.execute_chunking("mytag.test", es)
@@ -226,9 +226,9 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
 
       buffer_mock = flexmock(@i.buffer)
       buffer_mock.should_receive(:write).with({
-          m1 => [es1.to_msgpack_stream, 5],
-          m2 => [es2.to_msgpack_stream, 1],
-        }, bulk: true, enqueue: false).once
+          m1 => es1,
+          m2 => es2,
+        }, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, enqueue: false).once
 
       es = test_event_stream
       @i.execute_chunking("mytag.test", es)
@@ -253,9 +253,9 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
 
       buffer_mock = flexmock(@i.buffer)
       buffer_mock.should_receive(:write).with({
-          m1 => [es1.to_msgpack_stream(time_int: true), 5],
-          m2 => [es2.to_msgpack_stream(time_int: true), 1],
-        }, bulk: true, enqueue: false).once
+          m1 => es1,
+          m2 => es2,
+        }, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM_TIME_INT, enqueue: false).once
 
       es = test_event_stream
       @i.execute_chunking("mytag.test", es)
@@ -273,7 +273,7 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
       es = test_event_stream
 
       buffer_mock = flexmock(@i.buffer)
-      buffer_mock.should_receive(:write).once.with({m => [es.map{|t,r| [t,r].to_json }.join, es.size]}, bulk: true, enqueue: false)
+      buffer_mock.should_receive(:write).once.with({m => es.map{|t,r| [t,r].to_json }}, format: nil, enqueue: false)
 
       @i.execute_chunking("mytag.test", es)
     end
@@ -290,7 +290,7 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
       es = test_event_stream
 
       buffer_mock = flexmock(@i.buffer)
-      buffer_mock.should_receive(:write).once.with({m => [es.map{|t,r| [t,r].to_json }.join, es.size]}, bulk: true, enqueue: false)
+      buffer_mock.should_receive(:write).once.with({m => es.map{|t,r| [t,r].to_json}}, format: nil, enqueue: false)
 
       @i.execute_chunking("mytag.test", es)
     end
@@ -323,7 +323,7 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
           m1 => es1.map{|t,r| [t,r].to_json },
           m2 => es2.map{|t,r| [t,r].to_json },
           m3 => es3.map{|t,r| [t,r].to_json },
-        }, bulk: false, enqueue: false).once
+        }, enqueue: false).once
 
       es = test_event_stream
       @i.execute_chunking("mytag.test", es)
@@ -353,7 +353,7 @@ class StandardBufferedOutputTest < Test::Unit::TestCase
       buffer_mock.should_receive(:write).with({
           m1 => es1.map{|t,r| [t,r].to_json },
           m2 => es2.map{|t,r| [t,r].to_json },
-        }, bulk: false, enqueue: false).once
+        }, enqueue: false).once
 
       es = test_event_stream
       @i.execute_chunking("mytag.test", es)

data/test/plugin/test_parser_apache.rb

@@ -1,16 +1,20 @@
 require_relative '../helper'
 require 'fluent/test/driver/parser'
-require 'fluent/plugin/parser'
+require 'fluent/plugin/parser_apache'
 
 class ApacheParserTest < ::Test::Unit::TestCase
   def setup
     Fluent::Test.setup
-    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin.new_parser('apache'))
+  end
+
+  def create_driver(conf = {})
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::ApacheParser.new).configure(conf)
   end
 
   data('parse' => :parse, 'call' => :call)
   def test_call(method_name)
-    m = @parser.instance.method(method_name)
+    d = create_driver
+    m = d.instance.method(method_name)
     m.call('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777') { |time, record|
       assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
       assert_equal({
@@ -25,13 +29,13 @@ class ApacheParserTest < ::Test::Unit::TestCase
   end
 
   def test_parse_with_keep_time_key
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin.new_parser('apache'))
-    parser.instance.configure(
-                              'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
-                              'keep_time_key'=>'true',
-                              )
+    conf = {
+      'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+      'keep_time_key' => 'true',
+    }
+    d = create_driver(conf)
     text = '192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777'
-    parser.instance.parse(text) do |time, record|
+    d.instance.parse(text) do |_time, record|
       assert_equal "28/Feb/2013:12:00:00 +0900", record['time']
     end
   end

data/test/plugin/test_parser_apache_error.rb

@@ -1,12 +1,10 @@
 require_relative '../helper'
 require 'fluent/test/driver/parser'
-require 'fluent/plugin/parser'
+require 'fluent/plugin/parser_apache_error'
 
 class ApacheErrorParserTest < ::Test::Unit::TestCase
   def setup
     Fluent::Test.setup
-    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin.new_parser('apache_error'))
-    @parser.configure({})
     @expected = {
       'level' => 'error',
       'client' => '127.0.0.1',
@@ -14,22 +12,29 @@ class ApacheErrorParserTest < ::Test::Unit::TestCase
     }
   end
 
+  def create_driver
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::ApacheErrorParser.new).configure({})
+  end
+
   def test_parse
-    @parser.instance.parse('[Wed Oct 11 14:32:52 2000] [error] [client 127.0.0.1] client denied by server configuration') { |time, record|
+    d = create_driver
+    d.instance.parse('[Wed Oct 11 14:32:52 2000] [error] [client 127.0.0.1] client denied by server configuration') { |time, record|
       assert_equal(event_time('Wed Oct 11 14:32:52 2000'), time)
       assert_equal(@expected, record)
     }
   end
 
   def test_parse_with_pid
-    @parser.instance.parse('[Wed Oct 11 14:32:52 2000] [error] [pid 1000] [client 127.0.0.1] client denied by server configuration') { |time, record|
+    d = create_driver
+    d.instance.parse('[Wed Oct 11 14:32:52 2000] [error] [pid 1000] [client 127.0.0.1] client denied by server configuration') { |time, record|
       assert_equal(event_time('Wed Oct 11 14:32:52 2000'), time)
       assert_equal(@expected.merge('pid' => '1000'), record)
     }
   end
 
   def test_parse_without_client
-    @parser.instance.parse('[Wed Oct 11 14:32:52 2000] [notice] Apache/2.2.15 (Unix) DAV/2 configured -- resuming normal operations') { |time, record|
+    d = create_driver
+    d.instance.parse('[Wed Oct 11 14:32:52 2000] [notice] Apache/2.2.15 (Unix) DAV/2 configured -- resuming normal operations') { |time, record|
       assert_equal(event_time('Wed Oct 11 14:32:52 2000'), time)
       assert_equal({
                      'level' => 'notice',

data/test/plugin/test_parser_csv.rb

@@ -1,17 +1,20 @@
 require_relative '../helper'
 require 'fluent/test/driver/parser'
-require 'fluent/plugin/parser'
+require 'fluent/plugin/parser_csv'
 
 class CSVParserTest < ::Test::Unit::TestCase
   def setup
     Fluent::Test.setup
   end
 
+  def create_driver(conf={})
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::CSVParser).configure(conf)
+  end
+
   data('array param' => '["time","c","d"]', 'string param' => 'time,c,d')
   def test_parse(param)
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::CSVParser)
-    parser.configure('keys' => param, 'time_key' => 'time')
-    parser.instance.parse("2013/02/28 12:00:00,192.168.0.1,111") { |time, record|
+    d = create_driver('keys' => param, 'time_key' => 'time')
+    d.instance.parse("2013/02/28 12:00:00,192.168.0.1,111") { |time, record|
       assert_equal(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
       assert_equal({
                      'c' => '192.168.0.1',
@@ -24,9 +27,8 @@ class CSVParserTest < ::Test::Unit::TestCase
   def test_parse_without_time(param)
     time_at_start = Time.now.to_i
 
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::CSVParser)
-    parser.configure('keys' => param)
-    parser.instance.parse("192.168.0.1,111") { |time, record|
+    d = create_driver('keys' => param)
+    d.instance.parse("192.168.0.1,111") { |time, record|
       assert time && time >= time_at_start, "parser puts current time without time input"
       assert_equal({
                      'c' => '192.168.0.1',
@@ -34,10 +36,10 @@ class CSVParserTest < ::Test::Unit::TestCase
                    }, record)
     }
 
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::CSVParser)
-    parser.instance.estimate_current_event = false
-    parser.configure('keys' => param, 'time_key' => 'time')
-    parser.instance.parse("192.168.0.1,111") { |time, record|
+    d = Fluent::Test::Driver::Parser.new(Fluent::Plugin::CSVParser)
+    d.instance.estimate_current_event = false
+    d.configure('keys' => param)
+    d.instance.parse("192.168.0.1,111") { |time, record|
       assert_equal({
                      'c' => '192.168.0.1',
                      'd' => '111',
@@ -47,48 +49,56 @@ class CSVParserTest < ::Test::Unit::TestCase
   end
 
   def test_parse_with_keep_time_key
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::CSVParser)
-    parser.configure(
+    d = create_driver(
                      'keys'=>'time',
                      'time_key'=>'time',
                      'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
                      'keep_time_key'=>'true',
                      )
     text = '28/Feb/2013:12:00:00 +0900'
-    parser.instance.parse(text) do |time, record|
+    d.instance.parse(text) do |time, record|
       assert_equal text, record['time']
     end
   end
 
   data('array param' => '["a","b","c","d","e","f"]', 'string param' => 'a,b,c,d,e,f')
-  def test_parse_with_null_value_pattern
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::CSVParser)
-    parser.configure(
+  def test_parse_with_null_value_pattern(param)
+    d = create_driver(
                      'keys'=>param,
-                     'time_key'=>'time',
                      'null_value_pattern'=>'^(-|null|NULL)$'
                      )
-    parser.instance.parse("-,null,NULL,,--,nuLL") do |time, record|
+    d.instance.parse("-,null,NULL,,--,nuLL") do |time, record|
       assert_nil record['a']
       assert_nil record['b']
       assert_nil record['c']
-      assert_equal record['d'], ''
+      assert_nil record['d']
       assert_equal record['e'], '--'
       assert_equal record['f'], 'nuLL'
     end
   end
 
   data('array param' => '["a","b"]', 'string param' => 'a,b')
-  def test_parse_with_null_empty_string
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::CSVParser)
-    parser.configure(
+  def test_parse_with_null_empty_string(param)
+    d = create_driver(
                      'keys'=>param,
-                     'time_key'=>'time',
                      'null_empty_string'=>true
                      )
-    parser.instance.parse(", ") do |time, record|
+    d.instance.parse(", ") do |time, record|
       assert_nil record['a']
       assert_equal record['b'], ' '
     end
   end
+
+  data('array param' => '["a","b","c"]', 'string param' => 'a,b,c')
+  def test_parse_with_option_delimiter(param)
+    d = create_driver(
+                     'keys'=>param,
+                     'delimiter'=>' ',
+                     )
+    d.instance.parse("123 456 789") do |time, record|
+      assert_equal record['a'], '123'
+      assert_equal record['b'], '456'
+      assert_equal record['c'], '789'
+    end
+  end
 end

data/test/plugin/test_parser_nginx.rb

@@ -1,11 +1,10 @@
 require_relative '../helper'
 require 'fluent/test/driver/parser'
-require 'fluent/plugin/parser'
+require 'fluent/plugin/parser_nginx'
 
 class NginxParserTest < ::Test::Unit::TestCase
   def setup
     Fluent::Test.setup
-    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin.new_parser('nginx'))
     @expected = {
       'remote'  => '127.0.0.1',
       'host'    => '192.168.0.1',
@@ -19,22 +18,29 @@ class NginxParserTest < ::Test::Unit::TestCase
     }
   end
 
+  def create_driver
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::NginxParser.new).configure({})
+  end
+
   def test_parse
-    @parser.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
+    d = create_driver
+    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
       assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
       assert_equal(@expected, record)
     }
   end
 
   def test_parse_with_empty_included_path
-    @parser.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /a[ ]b HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
+    d = create_driver
+    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /a[ ]b HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
       assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
       assert_equal(@expected.merge('path' => '/a[ ]b'), record)
     }
   end
 
   def test_parse_without_http_version
-    @parser.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /" 200 777 "-" "Opera/12.0"') { |time, record|
+    d = create_driver
+    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /" 200 777 "-" "Opera/12.0"') { |time, record|
       assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
       assert_equal(@expected, record)
     }