fluentd 0.14.0 → 0.14.1

data/test/plugin/test_out_null.rb

@@ -0,0 +1,29 @@
+require_relative '../helper'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/out_null'
+
+class NullOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf = "")
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::NullOutput).configure(conf)
+  end
+
+  def test_configure
+    assert_nothing_raised do
+      create_driver
+    end
+  end
+
+  def test_process
+    d = create_driver
+    assert_nothing_raised do
+      d.run do
+        d.feed("test", Fluent::EventTime.now, {"test" => "null"})
+      end
+    end
+    assert_equal([], d.events(tag: "test"))
+  end
+end

data/test/plugin/test_out_relabel.rb

@@ -0,0 +1,28 @@
+require_relative '../helper'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/out_relabel'
+
+class RelabelOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def default_config
+    config_element('ROOT', '', {"@type"=>"relabel", "@label"=>"@RELABELED"})
+  end
+
+  def create_driver(conf = default_config)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::RelabelOutput).configure(conf)
+  end
+
+  def test_process
+    d = create_driver
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: 'test') do
+      d.feed(time, {"a"=>1})
+      d.feed(time, {"a"=>2})
+    end
+    assert_equal [["test", time, {"a"=>1}], ["test", time, {"a"=>2}]], d.events
+  end
+end

data/test/plugin/test_out_roundrobin.rb

@@ -1,5 +1,5 @@
 require_relative '../helper'
-require 'fluent/test'
+require 'fluent/test/driver/multi_output'
 require 'fluent/plugin/out_roundrobin'
 
 class RoundRobinOutputTest < Test::Unit::TestCase
@@ -7,6 +7,7 @@ class RoundRobinOutputTest < Test::Unit::TestCase
     def startup
       $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts'))
       require 'fluent/plugin/out_test'
+      require 'fluent/plugin/out_test2'
     end
 
     def shutdown
@@ -20,37 +21,37 @@ class RoundRobinOutputTest < Test::Unit::TestCase
 
   CONFIG = %[
     <store>
-      type test
+      @type test
       name c0
     </store>
     <store>
-      type test
+      @type test2
       name c1
     </store>
     <store>
-      type test
+      @type test
       name c2
     </store>
   ]
   CONFIG_WITH_WEIGHT = %[
     <store>
-      type test
+      @type test
       name c0
       weight 3
     </store>
     <store>
-      type test
+      @type test2
       name c1
       weight 3
     </store>
     <store>
-      type test
+      @type test
       name c2
     </store>
   ]
 
   def create_driver(conf = CONFIG)
-    Fluent::Test::OutputTestDriver.new(Fluent::RoundRobinOutput).configure(conf)
+    Fluent::Test::Driver::MultiOutput.new(Fluent::Plugin::RoundRobinOutput).configure(conf)
   end
 
   def test_configure
@@ -58,9 +59,16 @@ class RoundRobinOutputTest < Test::Unit::TestCase
 
     outputs = d.instance.outputs
     assert_equal 3, outputs.size
-    assert_equal Fluent::TestOutput, outputs[0].class
-    assert_equal Fluent::TestOutput, outputs[1].class
-    assert_equal Fluent::TestOutput, outputs[2].class
+
+    assert_equal Fluent::Plugin::TestOutput, outputs[0].class
+    assert_equal Fluent::Plugin::Test2Output, outputs[1].class
+    assert_equal Fluent::Plugin::TestOutput, outputs[2].class
+
+    assert !outputs[0].has_router?
+    assert outputs[1].has_router?
+    assert outputs[1].router
+    assert !outputs[2].has_router?
+
     assert_equal "c0", outputs[0].name
     assert_equal "c1", outputs[1].name
     assert_equal "c2", outputs[2].name
@@ -75,9 +83,11 @@ class RoundRobinOutputTest < Test::Unit::TestCase
 
     outputs = d.instance.outputs
     assert_equal 3, outputs.size
-    assert_equal Fluent::TestOutput, outputs[0].class
-    assert_equal Fluent::TestOutput, outputs[1].class
-    assert_equal Fluent::TestOutput, outputs[2].class
+
+    assert_equal Fluent::Plugin::TestOutput, outputs[0].class
+    assert_equal Fluent::Plugin::Test2Output, outputs[1].class
+    assert_equal Fluent::Plugin::TestOutput, outputs[2].class
+
     assert_equal "c0", outputs[0].name
     assert_equal "c1", outputs[1].name
     assert_equal "c2", outputs[2].name
@@ -89,15 +99,15 @@ class RoundRobinOutputTest < Test::Unit::TestCase
     assert_equal 1, weights[2]
   end
 
-  def test_emit
+  def test_events_feeded_to_plugins_by_roundrobin
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.run do
-      d.emit({"a"=>1}, time)
-      d.emit({"a"=>2}, time)
-      d.emit({"a"=>3}, time)
-      d.emit({"a"=>4}, time)
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: 'test') do
+      d.feed(time, {"a" => 1})
+      d.feed(time, {"a" => 2})
+      d.feed(time, {"a" => 3})
+      d.feed(time, {"a" => 4})
     end
 
     os = d.instance.outputs
@@ -114,19 +124,15 @@ class RoundRobinOutputTest < Test::Unit::TestCase
     assert_equal [
         [time, {"a"=>3}],
       ], os[2].events
-
-    d.instance.outputs.each {|o|
-      assert_not_nil o.router
-    }
   end
 
-  def test_emit_weighted
+  def test_events_feeded_with_specified_weights
     d = create_driver(CONFIG_WITH_WEIGHT)
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.run do
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: 'test') do
       14.times do |i|
-        d.emit({"a"=>i}, time)
+        d.feed(time, {"a" => i})
       end
     end
 

data/test/plugin/test_out_stdout.rb

@@ -37,10 +37,10 @@ class StdoutOutputTest < Test::Unit::TestCase
     time = event_time()
     out = capture_log do
       d.run(default_tag: 'test') do
-        d.feed(time, {'test' => 'test'})
+        d.feed(time, {'test' => 'test1'})
       end
     end
-    assert_equal "#{Time.at(time).localtime} test: {\"test\":\"test\"}\n", out
+    assert_equal "#{Time.at(time).localtime} test: {\"test\":\"test1\"}\n", out
 
     if data == 'yajl'
       # NOTE: Float::NAN is not jsonable
@@ -56,10 +56,10 @@ class StdoutOutputTest < Test::Unit::TestCase
     time = event_time()
     out = capture_log do
       d.run(default_tag: 'test') do
-        d.feed(time, {'test' => 'test'})
+        d.feed(time, {'test' => 'test2'})
       end
     end
-    assert_equal "#{Time.at(time).localtime} test: {\"test\"=>\"test\"}\n", out
+    assert_equal "#{Time.at(time).localtime} test: {\"test\"=>\"test2\"}\n", out
 
     # NOTE: Float::NAN is not jsonable, but hash string can output it.
     out = capture_log { d.feed('test', time, {'test' => Float::NAN}) }

data/test/plugin/test_output_as_buffered.rb

@@ -125,6 +125,57 @@ class BufferedOutputTest < Test::Unit::TestCase
     Timecop.return
   end
 
+  sub_test_case 'buffered output configured with many chunk keys' do
+    setup do
+      @stored_global_logger = $log
+      $log = Fluent::Test::TestLogger.new
+      @hash = {
+        'flush_mode' => 'interval',
+        'flush_thread_burst_interval' => 0.01,
+        'chunk_limit_size' => 1024,
+        'timekey' => 60,
+      }
+      @i = create_output(:buffered)
+    end
+    teardown do
+      $log = @stored_global_logger
+    end
+    test 'nothing are warned with less chunk keys' do
+      chunk_keys = 'time,key1,key2,key3'
+      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_keys,@hash)]))
+      logs = @i.log.out.logs.dup
+      @i.start
+      assert{ logs.select{|log| log.include?('[warn]') }.size == 0 }
+    end
+
+    test 'a warning reported with 4 chunk keys' do
+      chunk_keys = 'key1,key2,key3,key4'
+      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_keys,@hash)]))
+      logs = @i.log.out.logs.dup
+
+      @i.start # this calls `log.reset`... capturing logs about configure must be done before this line
+      assert_equal ['key1', 'key2', 'key3', 'key4'], @i.chunk_keys
+
+      assert{ logs.select{|log| log.include?('[warn]: many chunk keys specified, and it may cause too many chunks on your system.') }.size == 1 }
+    end
+
+    test 'a warning reported with 4 chunk keys including "tag"' do
+      chunk_keys = 'tag,key1,key2,key3'
+      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_keys,@hash)]))
+      logs = @i.log.out.logs.dup
+      @i.start # this calls `log.reset`... capturing logs about configure must be done before this line
+      assert{ logs.select{|log| log.include?('[warn]: many chunk keys specified, and it may cause too many chunks on your system.') }.size == 1 }
+    end
+
+    test 'time key is not included for warned chunk keys' do
+      chunk_keys = 'time,key1,key2,key3'
+      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_keys,@hash)]))
+      logs = @i.log.out.logs.dup
+      @i.start
+      assert{ logs.select{|log| log.include?('[warn]') }.size == 0 }
+    end
+  end
+
   sub_test_case 'buffered output feature without any buffer key, flush_mode: lazy' do
     setup do
       hash = {

data/test/plugin/test_output_as_buffered_secondary.rb

@@ -134,6 +134,19 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       end
     end
 
+    test 'uses same plugin type with primary if @type is missing in secondary' do
+      bufconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 30, 'retry_randomize' => false})
+      secconf = config_element('secondary','',{})
+      priconf = config_element('ROOT', '', {'@type' => 'output_secondary_test'}, [bufconf, secconf])
+      i = create_output()
+      assert_nothing_raised do
+        i.configure(priconf)
+      end
+      logs = i.log.out.logs
+      assert{ logs.empty? }
+      assert{ i.secondary.is_a? FluentPluginOutputAsBufferedSecondaryTest::DummyFullFeatureOutput }
+    end
+
     test 'warns if secondary plugin is different type from primary one' do
       priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 30, 'retry_randomize' => false})
       secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})

data/test/plugin/test_parser_apache.rb

@@ -0,0 +1,38 @@
+require_relative '../helper'
+require 'fluent/test/driver/parser'
+require 'fluent/plugin/parser'
+
+class ApacheParserTest < ::Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin.new_parser('apache'))
+  end
+
+  data('parse' => :parse, 'call' => :call)
+  def test_call(method_name)
+    m = @parser.instance.method(method_name)
+    m.call('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777') { |time, record|
+      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
+      assert_equal({
+                     'user'    => '-',
+                     'method'  => 'GET',
+                     'code'    => '200',
+                     'size'    => '777',
+                     'host'    => '192.168.0.1',
+                     'path'    => '/'
+                   }, record)
+    }
+  end
+
+  def test_parse_with_keep_time_key
+    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin.new_parser('apache'))
+    parser.instance.configure(
+                              'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
+                              'keep_time_key'=>'true',
+                              )
+    text = '192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777'
+    parser.instance.parse(text) do |time, record|
+      assert_equal "28/Feb/2013:12:00:00 +0900", record['time']
+    end
+  end
+end

data/test/plugin/test_parser_apache2.rb

@@ -0,0 +1,38 @@
+require_relative '../helper'
+require 'fluent/test/driver/parser'
+require 'fluent/plugin/parser'
+
+class Apache2ParserTest < ::Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::Apache2Parser)
+    @expected = {
+      'user'    => nil,
+      'method'  => 'GET',
+      'code'    => 200,
+      'size'    => 777,
+      'host'    => '192.168.0.1',
+      'path'    => '/',
+      'referer' => nil,
+      'agent'   => 'Opera/12.0'
+    }
+  end
+
+  def test_parse
+    @parser.instance.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
+      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
+      assert_equal(@expected, record)
+    }
+    assert_equal(Fluent::Plugin::Apache2Parser::REGEXP,
+                 @parser.instance.patterns['format'])
+    assert_equal(Fluent::Plugin::Apache2Parser::TIME_FORMAT,
+                 @parser.instance.patterns['time_format'])
+  end
+
+  def test_parse_without_http_version
+    @parser.instance.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET /" 200 777 "-" "Opera/12.0"') { |time, record|
+      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
+      assert_equal(@expected, record)
+    }
+  end
+end

data/test/plugin/test_parser_apache_error.rb

@@ -0,0 +1,40 @@
+require_relative '../helper'
+require 'fluent/test/driver/parser'
+require 'fluent/plugin/parser'
+
+class ApacheErrorParserTest < ::Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin.new_parser('apache_error'))
+    @parser.configure({})
+    @expected = {
+      'level' => 'error',
+      'client' => '127.0.0.1',
+      'message' => 'client denied by server configuration'
+    }
+  end
+
+  def test_parse
+    @parser.instance.parse('[Wed Oct 11 14:32:52 2000] [error] [client 127.0.0.1] client denied by server configuration') { |time, record|
+      assert_equal(event_time('Wed Oct 11 14:32:52 2000'), time)
+      assert_equal(@expected, record)
+    }
+  end
+
+  def test_parse_with_pid
+    @parser.instance.parse('[Wed Oct 11 14:32:52 2000] [error] [pid 1000] [client 127.0.0.1] client denied by server configuration') { |time, record|
+      assert_equal(event_time('Wed Oct 11 14:32:52 2000'), time)
+      assert_equal(@expected.merge('pid' => '1000'), record)
+    }
+  end
+
+  def test_parse_without_client
+    @parser.instance.parse('[Wed Oct 11 14:32:52 2000] [notice] Apache/2.2.15 (Unix) DAV/2 configured -- resuming normal operations') { |time, record|
+      assert_equal(event_time('Wed Oct 11 14:32:52 2000'), time)
+      assert_equal({
+                     'level' => 'notice',
+                     'message' => 'Apache/2.2.15 (Unix) DAV/2 configured -- resuming normal operations'
+                   }, record)
+    }
+  end
+end

data/test/plugin/test_parser_base.rb

@@ -0,0 +1,32 @@
+require_relative '../helper'
+require 'fluent/test/driver/parser'
+require 'fluent/plugin/parser'
+
+module ParserTest
+  class BaseParserTest < ::Test::Unit::TestCase
+    def setup
+      Fluent::Test.setup
+    end
+
+    def create_driver(conf={})
+      Fluent::Test::Driver::Parser.new(Fluent::Plugin::Parser).configure(conf)
+    end
+
+    def test_init
+      d = create_driver
+      assert_true d.instance.estimate_current_event
+    end
+
+    def test_configure_against_string_literal
+      d = create_driver('keep_time_key true')
+      assert_true d.instance.keep_time_key
+    end
+
+    def test_parse
+      d = create_driver
+      assert_raise NotImplementedError do
+        d.instance.parse('')
+      end
+    end
+  end
+end