fluentd 0.14.17-x86-mingw32 → 1.3.1-x86-mingw32

data/test/plugin_helper/test_child_process.rb

@@ -510,7 +510,7 @@ class ChildProcessTest < Test::Unit::TestCase
   unless Fluent.windows?
     test 'can specify subprocess name' do
       io = IO.popen([["cat", "caaaaaaaaaaat"], '-'])
-      process_naming_enabled = (open("|ps"){|_io| _io.readlines }.select{|line| line.include?("caaaaaaaaaaat") }.size > 0)
+      process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.select{|line| line.include?("caaaaaaaaaaat") }.size > 0)
       Process.kill(:TERM, io.pid) rescue nil
       io.close rescue nil
 
@@ -527,15 +527,15 @@ class ChildProcessTest < Test::Unit::TestCase
           m.lock
           ran = true
           pids << @d.child_process_id
-          proc_lines += open("|ps"){|_io| _io.readlines }
+          proc_lines += open("|ps opid,cmd"){|_io| _io.readlines }
           m.unlock
           readio.read
         end
         sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
         m.lock
         pid = pids.first
-        # 51358 ttys001    0:00.00 sleeper -e sleep 10
-        assert{ proc_lines.select{|line| line =~ /^\s*#{pid}\s/ }.first.strip.split(/\s+/)[3] == "sleeeeeeeeeper" }
+        # 16357 sleeeeeeeeeper -e sleep 10; puts "hello"
+        assert{ proc_lines.select{|line| line =~ /^\s*#{pid}\s/ }.first.strip.split(/\s+/)[1] == "sleeeeeeeeeper" }
         @d.stop; @d.shutdown; @d.close; @d.terminate
       end
     end

data/test/plugin_helper/test_compat_parameters.rb

@@ -328,4 +328,26 @@ class CompatParameterTest < Test::Unit::TestCase
       # TODO:
     end
   end
+
+  sub_test_case 'parser plugins' do
+    test 'syslog parser parameters' do
+      hash = {
+        'format' => 'syslog',
+        'message_format' => 'rfc5424',
+        'with_priority' => 'true',
+        'rfc5424_time_format' => '%Y'
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = DummyI0.new
+      @i.configure(conf)
+      @i.start
+      @i.after_start
+
+      parser = @i.parser
+      assert_kind_of(Fluent::Plugin::SyslogParser, parser)
+      assert_equal :rfc5424, parser.message_format
+      assert_equal true, parser.with_priority
+      assert_equal '%Y', parser.rfc5424_time_format
+    end
+  end
 end

data/test/plugin_helper/test_record_accessor.rb

@@ -0,0 +1,197 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/record_accessor'
+require 'fluent/plugin/base'
+
+require 'time'
+
+class RecordAccessorHelperTest < Test::Unit::TestCase
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :record_accessor
+  end
+
+  sub_test_case 'parse nested key expression' do
+    data('normal' => 'key1',
+         'space' => 'ke y2',
+         'dot key' => 'this.is.key3')
+    test 'parse single key' do |param|
+      result = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(param)
+      assert_equal param, result
+    end
+
+    test "nested bracket keys with dot" do
+      result = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter("$['key1']['this.is.key3']")
+      assert_equal ['key1', 'this.is.key3'], result
+    end
+
+    data('dot' => '$.key1.key2[0]',
+         'bracket' => "$['key1']['key2'][0]",
+         'bracket w/ double quotes' => '$["key1"]["key2"][0]')
+    test "nested keys ['key1', 'key2', 0]" do |param|
+      result = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(param)
+      assert_equal ['key1', 'key2', 0], result
+    end
+
+    data('bracket' => "$['key1'][0]['ke y2']",
+         'bracket w/ double quotes' => '$["key1"][0]["ke y2"]')
+    test "nested keys ['key1', 0, 'ke y2']" do |param|
+      result = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(param)
+      assert_equal ['key1', 0, 'ke y2'], result
+    end
+
+    data('dot' => '$.[0].key1.[1].key2',
+         'bracket' => "$[0]['key1'][1]['key2']",
+         'bracket w/ double quotes' => '$[0]["key1"][1]["key2"]')
+    test "nested keys [0, 'key1', 1, 'key2']" do |param|
+      result = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(param)
+      assert_equal [0, 'key1', 1, 'key2'], result
+    end
+
+    data("missing ']'" => "$['key1'",
+         "missing array index with dot" => "$.hello[]",
+         "missing array index with braket" => "$[]",
+         "more chars" => "$.key1[0]foo",
+         "whitespace char included key in dot notation" => "$.key[0].ke y",
+         "empty keys with dot" => "$.",
+         "empty keys with bracket" => "$[",
+         "mismatched quotes1" => "$['key1']['key2\"]",
+         "mismatched quotes2" => '$["key1"]["key2\']')
+    test 'invalid syntax' do |param|
+      assert_raise Fluent::ConfigError do
+        Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(param)
+      end
+    end
+  end
+
+  sub_test_case 'attr_reader :keys' do
+    setup do
+      @d = Dummy.new
+    end
+
+    data('normal' => 'key1',
+         'space' => 'ke y2',
+         'dot key' => 'this.is.key3')
+    test 'access single key' do |param|
+      accessor = @d.record_accessor_create(param)
+      assert_equal param, accessor.keys
+    end
+
+    test "nested bracket keys with dot" do
+      accessor = @d.record_accessor_create("$['key1']['this.is.key3']")
+      assert_equal ['key1','this.is.key3'], accessor.keys
+    end
+
+    data('dot' => '$.key1.key2[0]',
+         'bracket' => "$['key1']['key2'][0]",
+         'bracket w/ double quotes' => '$["key1"]["key2"][0]')
+    test "nested keys ['key1', 'key2', 0]" do |param|
+      accessor = @d.record_accessor_create(param)
+      assert_equal ['key1', 'key2', 0], accessor.keys
+    end
+
+    data('bracket' => "$['key1'][0]['ke y2']",
+         'bracket w/ double quotes' => '$["key1"][0]["ke y2"]')
+    test "nested keys ['key1', 0, 'ke y2']" do |param|
+      accessor = @d.record_accessor_create(param)
+      assert_equal ['key1', 0, 'ke y2'], accessor.keys
+    end
+  end
+
+  sub_test_case Fluent::PluginHelper::RecordAccessor::Accessor do
+    setup do
+      @d = Dummy.new
+    end
+
+    data('normal' => 'key1',
+         'space' => 'ke y2',
+         'dot key' => 'this.is.key3')
+    test 'access single key' do |param|
+      r = {'key1' => 'v1', 'ke y2' => 'v2', 'this.is.key3' => 'v3'}
+      accessor = @d.record_accessor_create(param)
+      assert_equal r[param], accessor.call(r)
+    end
+
+    test "access single dot key using bracket style" do
+      r = {'key1' => 'v1', 'ke y2' => 'v2', 'this.is.key3' => 'v3'}
+      accessor = @d.record_accessor_create('$["this.is.key3"]')
+      assert_equal 'v3', accessor.call(r)
+    end
+
+    test "nested bracket keys with dot" do
+      r = {'key1' => {'this.is.key3' => 'value'}}
+      accessor = @d.record_accessor_create("$['key1']['this.is.key3']")
+      assert_equal 'value', accessor.call(r)
+    end
+
+    data('dot' => '$.key1.key2[0]',
+         'bracket' => "$['key1']['key2'][0]",
+         'bracket w/ double quotes' => '$["key1"]["key2"][0]')
+    test "nested keys ['key1', 'key2', 0]" do |param|
+      r = {'key1' => {'key2' => [1, 2, 3]}}
+      accessor = @d.record_accessor_create(param)
+      assert_equal 1, accessor.call(r)
+    end
+
+    data('bracket' => "$['key1'][0]['ke y2']",
+         'bracket w/ double quotes' => '$["key1"][0]["ke y2"]')
+    test "nested keys ['key1', 0, 'ke y2']" do |param|
+      r = {'key1' => [{'ke y2' => "value"}]}
+      accessor = @d.record_accessor_create(param)
+      assert_equal 'value', accessor.call(r)
+    end
+
+    data("missing ']'" => "$['key1'",
+         "missing array index with dot" => "$.hello[]",
+         "missing array index with braket" => "$['hello'][]",
+         "whitespace char included key in dot notation" => "$.key[0].ke y",
+         "more chars" => "$.key1[0]foo",
+         "empty keys with dot" => "$.",
+         "empty keys with bracket" => "$[",
+         "mismatched quotes1" => "$['key1']['key2\"]",
+         "mismatched quotes2" => '$["key1"]["key2\']')
+    test 'invalid syntax' do |param|
+      assert_raise Fluent::ConfigError do
+        @d.record_accessor_create(param)
+      end
+    end
+  end
+
+  sub_test_case 'Fluent::PluginHelper::RecordAccessor::Accessor#delete' do
+    setup do
+      @d = Dummy.new
+    end
+
+    data('normal' => 'key1',
+         'space' => 'ke y2',
+         'dot key' => 'this.is.key3')
+    test 'delete top key' do |param|
+      r = {'key1' => 'v1', 'ke y2' => 'v2', 'this.is.key3' => 'v3'}
+      accessor = @d.record_accessor_create(param)
+      accessor.delete(r)
+      assert_not_include(r, param)
+    end
+
+    test "delete top key using bracket style" do
+      r = {'key1' => 'v1', 'ke y2' => 'v2', 'this.is.key3' => 'v3'}
+      accessor = @d.record_accessor_create('$["this.is.key3"]')
+      accessor.delete(r)
+      assert_not_include(r, 'this.is.key3')
+    end
+
+    data('bracket' => "$['key1'][0]['ke y2']",
+         'bracket w/ double quotes' => '$["key1"][0]["ke y2"]')
+    test "delete nested keys ['key1', 0, 'ke y2']" do |param|
+      r = {'key1' => [{'ke y2' => "value"}]}
+      accessor = @d.record_accessor_create(param)
+      accessor.delete(r)
+      assert_not_include(r['key1'][0], 'ke y2')
+    end
+
+    test "don't raise an error when unexpected record is coming" do
+      r = {'key1' => [{'key3' => "value"}]}
+      accessor = @d.record_accessor_create("$['key1']['key2']['key3']")
+      assert_nothing_raised do
+        assert_nil accessor.delete(r)
+      end
+    end
+  end
+end

data/test/plugin_helper/test_retry_state.rb

@@ -419,4 +419,24 @@ class RetryStateHelperTest < Test::Unit::TestCase
     assert_equal (dummy_current_time + 100), s.timeout_at
     assert_equal (dummy_current_time + timeout * 0.8), s.secondary_transition_at
   end
+
+  sub_test_case 'exponential backoff' do
+    test 'too big steps(check inf handling)' do
+      s = @d.retry_state_create(:t11, :exponential_backoff, 1, 300, randomize: false, forever: true, backoff_base: 2)
+      dummy_current_time = s.start
+      override_current_time(s, dummy_current_time)
+
+      i = 1
+      while i < 1027
+        if i >= 1025
+          # With this setting, 1025+ number causes inf in `calc_interval`, so 1024 value is used for next_time
+          assert_nothing_raised(FloatDomainError) { s.step }
+          assert_equal (dummy_current_time + (2 ** (1024 - 1))), s.next_time
+        else
+          s.step
+        end
+        i += 1
+      end
+    end
+  end
 end

data/test/plugin_helper/test_server.rb

@@ -776,6 +776,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
   def create_server_pair_signed_by_self(cert_path, private_key_path, passphrase)
     cert, key, _ = CertUtil.cert_option_generate_server_pair_self_signed(create_server_options)
     write_cert_and_key(cert_path, cert, private_key_path, key, passphrase)
+    return cert
   end
 
   def create_ca_pair_signed_by_self(cert_path, private_key_path, passphrase)
@@ -786,6 +787,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
   def create_server_pair_signed_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, passphrase)
     cert, key, _ = CertUtil.cert_option_generate_server_pair_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, create_server_options)
     write_cert_and_key(cert_path, cert, private_key_path, key, passphrase)
+    return cert
   end
 
   def create_server_pair_chained_with_root_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, passphrase)
@@ -842,6 +844,20 @@ class ServerPluginHelperTest < Test::Unit::TestCase
     sock.close rescue nil
   end
 
+  def assert_certificate(cert, expected_extensions)
+    get_extension = lambda do |oid|
+      cert.extensions.detect { |e| e.oid == oid }
+    end
+
+    assert_true cert.serial > 1
+    assert_equal 2, cert.version
+
+    expected_extensions.each do |ext|
+      expected_oid, expected_value = ext
+      assert_equal expected_value, get_extension.call(expected_oid).value
+    end
+  end
+
   sub_test_case '#server_create_tls with various certificate options' do
     setup do
       @d = Dummy.new # to get plugin not configured/started yet
@@ -896,7 +912,12 @@ class ServerPluginHelperTest < Test::Unit::TestCase
       test 'load self-signed cert/key pair (files), verified from clients using cert files' do |private_key_passphrase|
         cert_path = File.join(@server_cert_dir, "cert.pem")
         private_key_path = File.join(@certs_dir, "server.key.pem")
-        create_server_pair_signed_by_self(cert_path, private_key_path, private_key_passphrase)
+        cert = create_server_pair_signed_by_self(cert_path, private_key_path, private_key_passphrase)
+
+        assert_certificate(cert,[
+          ['basicConstraints', 'CA:FALSE'],
+          ['nsCertType', 'SSL Server']
+        ])
 
         tls_options = {
           protocol: :tls,
@@ -963,7 +984,14 @@ class ServerPluginHelperTest < Test::Unit::TestCase
 
         cert_path = File.join(@server_cert_dir, "cert.pem")
         private_key_path = File.join(@certs_dir, "server.key.pem")
-        create_server_pair_signed_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, private_key_passphrase)
+        cert = create_server_pair_signed_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, private_key_passphrase)
+
+        assert_certificate(cert,[
+            ['basicConstraints', 'CA:FALSE'],
+            ['nsCertType', 'SSL Server'],
+            ['keyUsage', 'Digital Signature, Key Encipherment'],
+            ['extendedKeyUsage', 'TLS Web Server Authentication']
+        ])
 
         tls_options = {
           protocol: :tls,

data/test/test_config.rb

@@ -53,7 +53,7 @@ class ConfigTest < Test::Unit::TestCase
       </elem2>
     ]
     write_config "#{TMP_DIR}/dir/config_test_9.conf", %[
-      k9 embeded
+      k9 embedded
       <elem3 name>
         nested nested_value
         include hoge
@@ -98,7 +98,7 @@ class ConfigTest < Test::Unit::TestCase
     elem2 = c.elements.find { |e| e.name == 'elem2' }
     assert_not_nil elem2
     assert_equal 'name', elem2.arg
-    assert_equal 'embeded', elem2['k9']
+    assert_equal 'embedded', elem2['k9']
     assert !elem2.has_key?('include')
 
     elem3 = elem2.elements.find { |e| e.name == 'elem3' }
@@ -145,7 +145,7 @@ class ConfigTest < Test::Unit::TestCase
     not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }
     assert_equal %w[pattern replace], not_fetched
 
-    # repeateadly accessing should not grow memory usage
+    # repeatedly accessing should not grow memory usage
     before_size = match_conf.unused.size
     10.times { match_conf['type'] }
     assert_equal before_size, match_conf.unused.size

data/test/test_configdsl.rb

@@ -65,13 +65,13 @@ match('aa')
     e0 = root.elements[0]
     assert_equal 'source', e0.name
     assert_equal '', e0.arg
-    assert_equal 'forward', e0['type']
+    assert_equal 'forward', e0['@type']
     assert_equal '24224', e0['port']
 
     e1 = root.elements[1]
     assert_equal 'match', e1.name
     assert_equal 'test.**', e1.arg
-    assert_equal 'forward', e1['type']
+    assert_equal 'forward', e1['@type']
     assert_equal '1s', e1['flush_interval']
     assert_equal 2, e1.elements.size
     e1s0 = e1.elements[0]

data/test/test_log.rb

@@ -379,6 +379,56 @@ class LogTest < Test::Unit::TestCase
     assert_equal(Fluent::Log::LEVEL_TRACE, log2.level)
   end
 
+  def test_format_json
+    logdev = @log_device
+    logger = ServerEngine::DaemonLogger.new(logdev)
+    log = Fluent::Log.new(logger)
+    log.format = :json
+    log.level = Fluent::Log::LEVEL_TRACE
+    log.trace "trace log"
+    log.debug "debug log"
+    log.info "info log"
+    log.warn "warn log"
+    log.error "error log"
+    log.fatal "fatal log"
+    expected = [
+      "#{@timestamp_str} [trace]: trace log\n",
+      "#{@timestamp_str} [debug]: debug log\n",
+      "#{@timestamp_str} [info]: info log\n",
+      "#{@timestamp_str} [warn]: warn log\n",
+      "#{@timestamp_str} [error]: error log\n",
+      "#{@timestamp_str} [fatal]: fatal log\n"
+    ]
+    assert_equal(expected, log.out.logs.map { |l|
+                   r = JSON.parse(l)
+                   "#{r['time']} [#{r['level']}]: #{r['message']}\n"
+                 })
+  end
+
+  def test_time_format
+    logdev = @log_device
+    logger = ServerEngine::DaemonLogger.new(logdev)
+    log = Fluent::Log.new(logger)
+    log.time_format = "%Y"
+    log.level = Fluent::Log::LEVEL_TRACE
+    log.trace "trace log"
+    log.debug "debug log"
+    log.info "info log"
+    log.warn "warn log"
+    log.error "error log"
+    log.fatal "fatal log"
+    timestamp_str = @timestamp.strftime("%Y")
+    expected = [
+      "#{timestamp_str} [trace]: trace log\n",
+      "#{timestamp_str} [debug]: debug log\n",
+      "#{timestamp_str} [info]: info log\n",
+      "#{timestamp_str} [warn]: warn log\n",
+      "#{timestamp_str} [error]: error log\n",
+      "#{timestamp_str} [fatal]: fatal log\n"
+    ]
+    assert_equal(expected, log.out.logs)
+  end
+
   def test_disable_events
     dl_opts = {}
     dl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE
@@ -443,7 +493,7 @@ class LogTest < Test::Unit::TestCase
     end
   end
 
-  def test_log_rotates_specifed_size_with_logdevio
+  def test_log_rotates_specified_size_with_logdevio
     with_timezone('utc') do
       rotate_age = 2
       rotate_size = 100