fluentd 0.14.17-x64-mingw32 → 1.3.1-x64-mingw32

data/test/plugin_helper/test_record_accessor.rb

@@ -0,0 +1,197 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/record_accessor'
+require 'fluent/plugin/base'
+
+require 'time'
+
+class RecordAccessorHelperTest < Test::Unit::TestCase
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :record_accessor
+  end
+
+  sub_test_case 'parse nested key expression' do
+    data('normal' => 'key1',
+         'space' => 'ke y2',
+         'dot key' => 'this.is.key3')
+    test 'parse single key' do |param|
+      result = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(param)
+      assert_equal param, result
+    end
+
+    test "nested bracket keys with dot" do
+      result = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter("$['key1']['this.is.key3']")
+      assert_equal ['key1', 'this.is.key3'], result
+    end
+
+    data('dot' => '$.key1.key2[0]',
+         'bracket' => "$['key1']['key2'][0]",
+         'bracket w/ double quotes' => '$["key1"]["key2"][0]')
+    test "nested keys ['key1', 'key2', 0]" do |param|
+      result = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(param)
+      assert_equal ['key1', 'key2', 0], result
+    end
+
+    data('bracket' => "$['key1'][0]['ke y2']",
+         'bracket w/ double quotes' => '$["key1"][0]["ke y2"]')
+    test "nested keys ['key1', 0, 'ke y2']" do |param|
+      result = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(param)
+      assert_equal ['key1', 0, 'ke y2'], result
+    end
+
+    data('dot' => '$.[0].key1.[1].key2',
+         'bracket' => "$[0]['key1'][1]['key2']",
+         'bracket w/ double quotes' => '$[0]["key1"][1]["key2"]')
+    test "nested keys [0, 'key1', 1, 'key2']" do |param|
+      result = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(param)
+      assert_equal [0, 'key1', 1, 'key2'], result
+    end
+
+    data("missing ']'" => "$['key1'",
+         "missing array index with dot" => "$.hello[]",
+         "missing array index with braket" => "$[]",
+         "more chars" => "$.key1[0]foo",
+         "whitespace char included key in dot notation" => "$.key[0].ke y",
+         "empty keys with dot" => "$.",
+         "empty keys with bracket" => "$[",
+         "mismatched quotes1" => "$['key1']['key2\"]",
+         "mismatched quotes2" => '$["key1"]["key2\']')
+    test 'invalid syntax' do |param|
+      assert_raise Fluent::ConfigError do
+        Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(param)
+      end
+    end
+  end
+
+  sub_test_case 'attr_reader :keys' do
+    setup do
+      @d = Dummy.new
+    end
+
+    data('normal' => 'key1',
+         'space' => 'ke y2',
+         'dot key' => 'this.is.key3')
+    test 'access single key' do |param|
+      accessor = @d.record_accessor_create(param)
+      assert_equal param, accessor.keys
+    end
+
+    test "nested bracket keys with dot" do
+      accessor = @d.record_accessor_create("$['key1']['this.is.key3']")
+      assert_equal ['key1','this.is.key3'], accessor.keys
+    end
+
+    data('dot' => '$.key1.key2[0]',
+         'bracket' => "$['key1']['key2'][0]",
+         'bracket w/ double quotes' => '$["key1"]["key2"][0]')
+    test "nested keys ['key1', 'key2', 0]" do |param|
+      accessor = @d.record_accessor_create(param)
+      assert_equal ['key1', 'key2', 0], accessor.keys
+    end
+
+    data('bracket' => "$['key1'][0]['ke y2']",
+         'bracket w/ double quotes' => '$["key1"][0]["ke y2"]')
+    test "nested keys ['key1', 0, 'ke y2']" do |param|
+      accessor = @d.record_accessor_create(param)
+      assert_equal ['key1', 0, 'ke y2'], accessor.keys
+    end
+  end
+
+  sub_test_case Fluent::PluginHelper::RecordAccessor::Accessor do
+    setup do
+      @d = Dummy.new
+    end
+
+    data('normal' => 'key1',
+         'space' => 'ke y2',
+         'dot key' => 'this.is.key3')
+    test 'access single key' do |param|
+      r = {'key1' => 'v1', 'ke y2' => 'v2', 'this.is.key3' => 'v3'}
+      accessor = @d.record_accessor_create(param)
+      assert_equal r[param], accessor.call(r)
+    end
+
+    test "access single dot key using bracket style" do
+      r = {'key1' => 'v1', 'ke y2' => 'v2', 'this.is.key3' => 'v3'}
+      accessor = @d.record_accessor_create('$["this.is.key3"]')
+      assert_equal 'v3', accessor.call(r)
+    end
+
+    test "nested bracket keys with dot" do
+      r = {'key1' => {'this.is.key3' => 'value'}}
+      accessor = @d.record_accessor_create("$['key1']['this.is.key3']")
+      assert_equal 'value', accessor.call(r)
+    end
+
+    data('dot' => '$.key1.key2[0]',
+         'bracket' => "$['key1']['key2'][0]",
+         'bracket w/ double quotes' => '$["key1"]["key2"][0]')
+    test "nested keys ['key1', 'key2', 0]" do |param|
+      r = {'key1' => {'key2' => [1, 2, 3]}}
+      accessor = @d.record_accessor_create(param)
+      assert_equal 1, accessor.call(r)
+    end
+
+    data('bracket' => "$['key1'][0]['ke y2']",
+         'bracket w/ double quotes' => '$["key1"][0]["ke y2"]')
+    test "nested keys ['key1', 0, 'ke y2']" do |param|
+      r = {'key1' => [{'ke y2' => "value"}]}
+      accessor = @d.record_accessor_create(param)
+      assert_equal 'value', accessor.call(r)
+    end
+
+    data("missing ']'" => "$['key1'",
+         "missing array index with dot" => "$.hello[]",
+         "missing array index with braket" => "$['hello'][]",
+         "whitespace char included key in dot notation" => "$.key[0].ke y",
+         "more chars" => "$.key1[0]foo",
+         "empty keys with dot" => "$.",
+         "empty keys with bracket" => "$[",
+         "mismatched quotes1" => "$['key1']['key2\"]",
+         "mismatched quotes2" => '$["key1"]["key2\']')
+    test 'invalid syntax' do |param|
+      assert_raise Fluent::ConfigError do
+        @d.record_accessor_create(param)
+      end
+    end
+  end
+
+  sub_test_case 'Fluent::PluginHelper::RecordAccessor::Accessor#delete' do
+    setup do
+      @d = Dummy.new
+    end
+
+    data('normal' => 'key1',
+         'space' => 'ke y2',
+         'dot key' => 'this.is.key3')
+    test 'delete top key' do |param|
+      r = {'key1' => 'v1', 'ke y2' => 'v2', 'this.is.key3' => 'v3'}
+      accessor = @d.record_accessor_create(param)
+      accessor.delete(r)
+      assert_not_include(r, param)
+    end
+
+    test "delete top key using bracket style" do
+      r = {'key1' => 'v1', 'ke y2' => 'v2', 'this.is.key3' => 'v3'}
+      accessor = @d.record_accessor_create('$["this.is.key3"]')
+      accessor.delete(r)
+      assert_not_include(r, 'this.is.key3')
+    end
+
+    data('bracket' => "$['key1'][0]['ke y2']",
+         'bracket w/ double quotes' => '$["key1"][0]["ke y2"]')
+    test "delete nested keys ['key1', 0, 'ke y2']" do |param|
+      r = {'key1' => [{'ke y2' => "value"}]}
+      accessor = @d.record_accessor_create(param)
+      accessor.delete(r)
+      assert_not_include(r['key1'][0], 'ke y2')
+    end
+
+    test "don't raise an error when unexpected record is coming" do
+      r = {'key1' => [{'key3' => "value"}]}
+      accessor = @d.record_accessor_create("$['key1']['key2']['key3']")
+      assert_nothing_raised do
+        assert_nil accessor.delete(r)
+      end
+    end
+  end
+end

data/test/plugin_helper/test_retry_state.rb

@@ -419,4 +419,24 @@ class RetryStateHelperTest < Test::Unit::TestCase
     assert_equal (dummy_current_time + 100), s.timeout_at
     assert_equal (dummy_current_time + timeout * 0.8), s.secondary_transition_at
   end
+
+  sub_test_case 'exponential backoff' do
+    test 'too big steps(check inf handling)' do
+      s = @d.retry_state_create(:t11, :exponential_backoff, 1, 300, randomize: false, forever: true, backoff_base: 2)
+      dummy_current_time = s.start
+      override_current_time(s, dummy_current_time)
+
+      i = 1
+      while i < 1027
+        if i >= 1025
+          # With this setting, 1025+ number causes inf in `calc_interval`, so 1024 value is used for next_time
+          assert_nothing_raised(FloatDomainError) { s.step }
+          assert_equal (dummy_current_time + (2 ** (1024 - 1))), s.next_time
+        else
+          s.step
+        end
+        i += 1
+      end
+    end
+  end
 end

data/test/plugin_helper/test_server.rb

@@ -776,6 +776,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
   def create_server_pair_signed_by_self(cert_path, private_key_path, passphrase)
     cert, key, _ = CertUtil.cert_option_generate_server_pair_self_signed(create_server_options)
     write_cert_and_key(cert_path, cert, private_key_path, key, passphrase)
+    return cert
   end
 
   def create_ca_pair_signed_by_self(cert_path, private_key_path, passphrase)
@@ -786,6 +787,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
   def create_server_pair_signed_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, passphrase)
     cert, key, _ = CertUtil.cert_option_generate_server_pair_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, create_server_options)
     write_cert_and_key(cert_path, cert, private_key_path, key, passphrase)
+    return cert
   end
 
   def create_server_pair_chained_with_root_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, passphrase)
@@ -842,6 +844,20 @@ class ServerPluginHelperTest < Test::Unit::TestCase
     sock.close rescue nil
   end
 
+  def assert_certificate(cert, expected_extensions)
+    get_extension = lambda do |oid|
+      cert.extensions.detect { |e| e.oid == oid }
+    end
+
+    assert_true cert.serial > 1
+    assert_equal 2, cert.version
+
+    expected_extensions.each do |ext|
+      expected_oid, expected_value = ext
+      assert_equal expected_value, get_extension.call(expected_oid).value
+    end
+  end
+
   sub_test_case '#server_create_tls with various certificate options' do
     setup do
       @d = Dummy.new # to get plugin not configured/started yet
@@ -896,7 +912,12 @@ class ServerPluginHelperTest < Test::Unit::TestCase
       test 'load self-signed cert/key pair (files), verified from clients using cert files' do |private_key_passphrase|
         cert_path = File.join(@server_cert_dir, "cert.pem")
         private_key_path = File.join(@certs_dir, "server.key.pem")
-        create_server_pair_signed_by_self(cert_path, private_key_path, private_key_passphrase)
+        cert = create_server_pair_signed_by_self(cert_path, private_key_path, private_key_passphrase)
+
+        assert_certificate(cert,[
+          ['basicConstraints', 'CA:FALSE'],
+          ['nsCertType', 'SSL Server']
+        ])
 
         tls_options = {
           protocol: :tls,
@@ -963,7 +984,14 @@ class ServerPluginHelperTest < Test::Unit::TestCase
 
         cert_path = File.join(@server_cert_dir, "cert.pem")
         private_key_path = File.join(@certs_dir, "server.key.pem")
-        create_server_pair_signed_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, private_key_passphrase)
+        cert = create_server_pair_signed_by_ca(ca_cert_path, ca_key_path, ca_key_passphrase, cert_path, private_key_path, private_key_passphrase)
+
+        assert_certificate(cert,[
+            ['basicConstraints', 'CA:FALSE'],
+            ['nsCertType', 'SSL Server'],
+            ['keyUsage', 'Digital Signature, Key Encipherment'],
+            ['extendedKeyUsage', 'TLS Web Server Authentication']
+        ])
 
         tls_options = {
           protocol: :tls,

data/test/test_config.rb

@@ -53,7 +53,7 @@ class ConfigTest < Test::Unit::TestCase
       </elem2>
     ]
     write_config "#{TMP_DIR}/dir/config_test_9.conf", %[
-      k9 embeded
+      k9 embedded
       <elem3 name>
         nested nested_value
         include hoge
@@ -98,7 +98,7 @@ class ConfigTest < Test::Unit::TestCase
     elem2 = c.elements.find { |e| e.name == 'elem2' }
     assert_not_nil elem2
     assert_equal 'name', elem2.arg
-    assert_equal 'embeded', elem2['k9']
+    assert_equal 'embedded', elem2['k9']
     assert !elem2.has_key?('include')
 
     elem3 = elem2.elements.find { |e| e.name == 'elem3' }
@@ -145,7 +145,7 @@ class ConfigTest < Test::Unit::TestCase
     not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }
     assert_equal %w[pattern replace], not_fetched
 
-    # repeateadly accessing should not grow memory usage
+    # repeatedly accessing should not grow memory usage
     before_size = match_conf.unused.size
     10.times { match_conf['type'] }
     assert_equal before_size, match_conf.unused.size

data/test/test_configdsl.rb

@@ -65,13 +65,13 @@ match('aa')
     e0 = root.elements[0]
     assert_equal 'source', e0.name
     assert_equal '', e0.arg
-    assert_equal 'forward', e0['type']
+    assert_equal 'forward', e0['@type']
     assert_equal '24224', e0['port']
 
     e1 = root.elements[1]
     assert_equal 'match', e1.name
     assert_equal 'test.**', e1.arg
-    assert_equal 'forward', e1['type']
+    assert_equal 'forward', e1['@type']
     assert_equal '1s', e1['flush_interval']
     assert_equal 2, e1.elements.size
     e1s0 = e1.elements[0]

data/test/test_log.rb

@@ -379,6 +379,56 @@ class LogTest < Test::Unit::TestCase
     assert_equal(Fluent::Log::LEVEL_TRACE, log2.level)
   end
 
+  def test_format_json
+    logdev = @log_device
+    logger = ServerEngine::DaemonLogger.new(logdev)
+    log = Fluent::Log.new(logger)
+    log.format = :json
+    log.level = Fluent::Log::LEVEL_TRACE
+    log.trace "trace log"
+    log.debug "debug log"
+    log.info "info log"
+    log.warn "warn log"
+    log.error "error log"
+    log.fatal "fatal log"
+    expected = [
+      "#{@timestamp_str} [trace]: trace log\n",
+      "#{@timestamp_str} [debug]: debug log\n",
+      "#{@timestamp_str} [info]: info log\n",
+      "#{@timestamp_str} [warn]: warn log\n",
+      "#{@timestamp_str} [error]: error log\n",
+      "#{@timestamp_str} [fatal]: fatal log\n"
+    ]
+    assert_equal(expected, log.out.logs.map { |l|
+                   r = JSON.parse(l)
+                   "#{r['time']} [#{r['level']}]: #{r['message']}\n"
+                 })
+  end
+
+  def test_time_format
+    logdev = @log_device
+    logger = ServerEngine::DaemonLogger.new(logdev)
+    log = Fluent::Log.new(logger)
+    log.time_format = "%Y"
+    log.level = Fluent::Log::LEVEL_TRACE
+    log.trace "trace log"
+    log.debug "debug log"
+    log.info "info log"
+    log.warn "warn log"
+    log.error "error log"
+    log.fatal "fatal log"
+    timestamp_str = @timestamp.strftime("%Y")
+    expected = [
+      "#{timestamp_str} [trace]: trace log\n",
+      "#{timestamp_str} [debug]: debug log\n",
+      "#{timestamp_str} [info]: info log\n",
+      "#{timestamp_str} [warn]: warn log\n",
+      "#{timestamp_str} [error]: error log\n",
+      "#{timestamp_str} [fatal]: fatal log\n"
+    ]
+    assert_equal(expected, log.out.logs)
+  end
+
   def test_disable_events
     dl_opts = {}
     dl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE
@@ -443,7 +493,7 @@ class LogTest < Test::Unit::TestCase
     end
   end
 
-  def test_log_rotates_specifed_size_with_logdevio
+  def test_log_rotates_specified_size_with_logdevio
     with_timezone('utc') do
       rotate_age = 2
       rotate_size = 100

data/test/test_root_agent.rb

@@ -1,6 +1,7 @@
 require_relative 'helper'
 require 'fluent/event_router'
 require 'fluent/system_config'
+require 'timecop'
 require_relative 'test_plugin_classes'
 
 class RootAgentTest < ::Test::Unit::TestCase
@@ -609,6 +610,38 @@ EOC
     end
   end
 
+  sub_test_case 'configure emit_error_interval' do
+    setup do
+      system_config = SystemConfig.new
+      system_config.emit_error_log_interval = 30
+      @ra = RootAgent.new(log: $log, system_config: system_config)
+      stub(Engine).root_agent { @ra }
+      @ra.log.out.reset
+      one_minute_ago = Time.now.to_i - 60
+      Timecop.freeze(one_minute_ago)
+    end
+
+    teardown do
+      Timecop.return
+    end
+
+    test 'suppresses errors' do
+      mock(@ra.log).warn_backtrace()
+      e = StandardError.new('standard error')
+      begin
+        @ra.handle_emits_error("tag", nil, e)
+      rescue
+      end
+
+      begin
+      @ra.handle_emits_error("tag", nil, e)
+      rescue
+      end
+
+      assert_equal 1, @ra.log.out.logs.size
+    end
+  end
+
   sub_test_case 'configured at worker2 with 4 workers environment' do
     setup do
       ENV['SERVERENGINE_WORKER_ID'] = '2'

data/test/test_supervisor.rb

@@ -77,6 +77,46 @@ class SupervisorTest < ::Test::Unit::TestCase
     $log.out.reset
   end
 
+  def test_read_config_with_multibyte_string
+    tmp_path = "#{TMP_DIR}/dir/test_multibyte_config.conf"
+    conf_str = %[
+<source>
+  @type forward
+  @id forward_input
+  @label @INPUT
+</source>
+<label @INPUT>
+  <filter>
+    @type record_transformer
+    <record>
+      message こんにちは. ${record["name"]} has made a order of ${record["item"]} just now.
+    </record>
+  </filter>
+  <match>
+    @type stdout
+  </match>
+</label>
+]
+    FileUtils.mkdir_p(File.dirname(tmp_path))
+    File.open(tmp_path, "w:utf-8") {|file| file.write(conf_str) }
+
+    opts = Fluent::Supervisor.default_options
+    sv = Fluent::Supervisor.new(opts)
+
+    use_v1_config = {}
+    use_v1_config['use_v1_config'] = true
+
+    sv.instance_variable_set(:@config_path, tmp_path)
+    sv.instance_variable_set(:@use_v1_config, use_v1_config)
+    sv.send(:read_config)
+
+    conf = sv.instance_variable_get(:@conf)
+    label = conf.elements.detect {|e| e.name == "label" }
+    filter = label.elements.detect {|e| e.name == "filter" }
+    record_transformer = filter.elements.detect {|e| e.name = "record_transformer" }
+    assert_equal(Encoding::UTF_8, record_transformer["message"].encoding)
+  end
+
   def test_system_config
     opts = Fluent::Supervisor.default_options
     sv = Fluent::Supervisor.new(opts)
@@ -90,6 +130,21 @@ class SupervisorTest < ::Test::Unit::TestCase
   process_name "process_name"
   log_level info
   root_dir #{TMP_ROOT_DIR}
+  <log>
+    format json
+    time_format %Y
+  </log>
+  <counter_server>
+    bind 127.0.0.1
+    port 24321
+    scope server1
+    backup_path /tmp/backup
+  </counter_server>
+  <counter_client>
+    host 127.0.0.1
+    port 24321
+    timeout 2
+  </counter_client>
 </system>
     EOC
     conf = Fluent::Config.parse(conf_data, "(test)", "(test_dir)", true)
@@ -105,6 +160,17 @@ class SupervisorTest < ::Test::Unit::TestCase
     assert_equal "process_name", sys_conf.process_name
     assert_equal 2, sys_conf.log_level
     assert_equal TMP_ROOT_DIR, sys_conf.root_dir
+    assert_equal :json, sys_conf.log.format
+    assert_equal '%Y', sys_conf.log.time_format
+    counter_server = sys_conf.counter_server
+    assert_equal '127.0.0.1', counter_server.bind
+    assert_equal 24321, counter_server.port
+    assert_equal 'server1', counter_server.scope
+    assert_equal '/tmp/backup', counter_server.backup_path
+    counter_client = sys_conf.counter_client
+    assert_equal '127.0.0.1', counter_client.host
+    assert_equal 24321, counter_client.port
+    assert_equal 2, counter_client.timeout
   end
 
   def test_main_process_signal_handlers
@@ -319,6 +385,45 @@ class SupervisorTest < ::Test::Unit::TestCase
     assert_equal Fluent::Log::LEVEL_INFO, se_config[:log_level]
   end
 
+  def test_load_config_with_multibyte_string
+    tmp_path = "#{TMP_DIR}/dir/test_multibyte_config.conf"
+    conf_str = %[
+<source>
+  @type forward
+  @id forward_input
+  @label @INPUT
+</source>
+<label @INPUT>
+  <filter>
+    @type record_transformer
+    <record>
+      message こんにちは. ${record["name"]} has made a order of ${record["item"]} just now.
+    </record>
+  </filter>
+  <match>
+    @type stdout
+  </match>
+</label>
+]
+    FileUtils.mkdir_p(File.dirname(tmp_path))
+    File.open(tmp_path, "w:utf-8") {|file| file.write(conf_str) }
+
+    params = {}
+    params['workers'] = 1
+    params['use_v1_config'] = true
+    params['log_path'] = 'test/tmp/supervisor/log'
+    params['suppress_repeated_stacktrace'] = true
+    params['log_level'] = Fluent::Log::LEVEL_INFO
+    load_config_proc =  Proc.new { Fluent::Supervisor.load_config(tmp_path, params) }
+
+    se_config = load_config_proc.call
+    conf = se_config[:fluentd_conf]
+    label = conf.elements.detect {|e| e.name == "label" }
+    filter = label.elements.detect {|e| e.name == "filter" }
+    record_transformer = filter.elements.detect {|e| e.name = "record_transformer" }
+    assert_equal(Encoding::UTF_8, record_transformer["message"].encoding)
+  end
+
   def test_logger
     opts = Fluent::Supervisor.default_options
     sv = Fluent::Supervisor.new(opts)