fluentd 1.7.4-x64-mingw32 → 1.8.0-x64-mingw32

data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb

@@ -0,0 +1,21 @@
+require_relative '../../helper'
+require 'fluent/plugin_helper/service_discovery/round_robin_balancer'
+
+class TestRoundRobinBalancer < ::Test::Unit::TestCase
+  test 'select_service' do
+    rrb = Fluent::PluginHelper::ServiceDiscovery::RoundRobinBalancer.new
+    rrb.rebalance([1, 2, 3])
+
+    rrb.select_service { |n| assert_equal 1, n }
+    rrb.select_service { |n| assert_equal 2, n }
+    rrb.select_service { |n| assert_equal 3, n }
+    rrb.select_service { |n| assert_equal 1, n }
+    rrb.select_service { |n| assert_equal 2, n }
+    rrb.select_service { |n| assert_equal 3, n }
+    rrb.rebalance([1, 2, 3, 4])
+    rrb.select_service { |n| assert_equal 1, n }
+    rrb.select_service { |n| assert_equal 2, n }
+    rrb.select_service { |n| assert_equal 3, n }
+    rrb.select_service { |n| assert_equal 4, n }
+  end
+end

data/test/plugin_helper/test_server.rb

@@ -538,6 +538,19 @@ class ServerPluginHelperTest < Test::Unit::TestCase
       assert_equal ["yayfoo\n", "yayfoo\n", "yayfoo\n"], lines
       assert_equal ["closed", "closed", "closed"], callback_results
     end
+
+    test 'can listen IPv4 / IPv6 together' do
+      omit "IPv6 unavailable here" unless ipv6_enabled?
+
+      assert_nothing_raised do
+        @d.server_create_tcp(:s_ipv4, PORT, bind: '0.0.0.0', shared: false) do |data, conn|
+          # ...
+        end
+        @d.server_create_tcp(:s_ipv6, PORT, bind: '::', shared: false) do |data, conn|
+          # ...
+        end
+      end
+    end
   end
 
   sub_test_case '#server_create_udp' do

data/test/plugin_helper/test_service_discovery.rb

@@ -0,0 +1,72 @@
+require_relative '../helper'
+require 'flexmock/test_unit'
+require 'fluent/plugin_helper/service_discovery'
+require 'fluent/plugin/output'
+
+class ServiceDiscoveryHelper < Test::Unit::TestCase
+  PORT = unused_port
+  NULL_LOGGER = Logger.new(nil)
+
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :service_discovery
+
+    # Make these mehtod public
+    def service_discovery_create_manager(title, configurations:, load_balancer: nil, custom_build_method: nil, interval: 3)
+      super
+    end
+
+    def discovery_manager
+      super
+    end
+  end
+
+  setup do
+    @sd_file_dir = File.expand_path('../plugin/data/sd_file', __dir__)
+
+    @d = nil
+  end
+
+  teardown do
+    if @d
+      @d.stop unless @d.stopped?
+      @d.shutdown unless @d.shutdown?
+      @d.close unless @d.closed?
+      @d.terminate unless @d.terminated?
+    end
+  end
+
+  test 'start discovery manager' do
+    d = @d = Dummy.new
+
+    d.service_discovery_create_manager(
+      :service_discovery_helper_test,
+      configurations: [{ type: :static, conf:  config_element('root', '', {}, [config_element('service', '', { 'host' => '127.0.0.1', 'port' => '1234' })]) }],
+    )
+
+    assert_true !!d.discovery_manager
+
+    mock.proxy(d.discovery_manager).start.once
+    mock.proxy(d).timer_execute(:service_discovery_helper_test, anything).never
+
+    d.start
+
+    services = d.discovery_manager.services
+    assert_equal 1, services.size
+    assert_equal '127.0.0.1', services[0].host
+    assert_equal 1234, services[0].port
+  end
+
+  test 'call timer_execute if dynamic configuration' do
+    d = @d = Dummy.new
+
+    d.service_discovery_create_manager(
+      :service_discovery_helper_test,
+      configurations: [{ type: :file, conf: config_element('file_config', '', { 'path' => File.join(@sd_file_dir, 'config.yml') }) }],
+    )
+
+    assert_true !!d.discovery_manager
+    mock.proxy(d.discovery_manager).start.once
+    mock(d).timer_execute(:service_discovery_helper_test, anything).once
+    d.start
+  end
+end

data/test/test_event.rb

@@ -74,7 +74,7 @@ module EventTest
 
     test 'to_msgpack_stream' do
       stream = @es.to_msgpack_stream
-      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|
         assert_equal @time, time
         assert_equal @record, record
       }
@@ -82,7 +82,7 @@ module EventTest
 
     test 'to_msgpack_stream with time_int argument' do
       stream = @es.to_msgpack_stream(time_int: true)
-      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|
         assert_equal @time.to_i, time
         assert_equal @record, record
       }
@@ -90,7 +90,7 @@ module EventTest
 
     test 'to_compressed_msgpack_stream' do
       stream = @es.to_compressed_msgpack_stream
-      Fluent::Engine.msgpack_factory.unpacker.feed_each(decompress(stream)) { |time, record|
+      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(decompress(stream)) { |time, record|
         assert_equal @time, time
         assert_equal @record, record
       }
@@ -98,7 +98,7 @@ module EventTest
 
     test 'to_compressed_msgpack_stream with time_int argument' do
       stream = @es.to_compressed_msgpack_stream(time_int: true)
-      Fluent::Engine.msgpack_factory.unpacker.feed_each(decompress(stream)) { |time, record|
+      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(decompress(stream)) { |time, record|
         assert_equal @time.to_i, time
         assert_equal @record, record
       }
@@ -174,7 +174,7 @@ module EventTest
     test 'to_msgpack_stream' do
       i = 0
       stream = @es.to_msgpack_stream
-      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|
         assert_equal @times[i], time
         assert_equal @records[i], record
         i += 1
@@ -185,7 +185,7 @@ module EventTest
       i = 0
       compressed_stream = @es.to_compressed_msgpack_stream
       stream = decompress(compressed_stream)
-      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|
         assert_equal @times[i], time
         assert_equal @records[i], record
         i += 1
@@ -196,7 +196,7 @@ module EventTest
       i = 0
       compressed_stream = @es.to_compressed_msgpack_stream(time_int: true)
       stream = decompress(compressed_stream)
-      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|
         assert_equal @times[i].to_i, time
         assert_equal @records[i], record
         i += 1
@@ -276,7 +276,7 @@ module EventTest
     test 'to_msgpack_stream' do
       i = 0
       stream = @es.to_msgpack_stream
-      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|
         assert_equal @times[i], time
         assert_equal @records[i], record
         i += 1
@@ -287,7 +287,7 @@ module EventTest
       i = 0
       compressed_stream = @es.to_compressed_msgpack_stream
       stream = decompress(compressed_stream)
-      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|
         assert_equal @times[i], time
         assert_equal @records[i], record
         i += 1
@@ -298,7 +298,7 @@ module EventTest
       i = 0
       compressed_stream = @es.to_compressed_msgpack_stream(time_int: true)
       stream = decompress(compressed_stream)
-      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|
         assert_equal @times[i].to_i, time
         assert_equal @records[i], record
         i += 1
@@ -312,7 +312,7 @@ module EventTest
     include Fluent::Plugin::Compressable
 
     def setup
-      pk = Fluent::Engine.msgpack_factory.packer
+      pk = Fluent::MessagePackFactory.msgpack_packer
       time = Engine.now
       @times = [Fluent::EventTime.new(time.sec), Fluent::EventTime.new(time.sec + 1)]
       @records = [{'k' => 'v1', 'n' => 1}, {'k' => 'v2', 'n' => 2}]
@@ -384,7 +384,7 @@ module EventTest
     test 'to_msgpack_stream' do
       i = 0
       stream = @es.to_msgpack_stream
-      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|
         assert_equal @times[i], time
         assert_equal @records[i], record
         i += 1
@@ -395,7 +395,7 @@ module EventTest
       i = 0
       compressed_stream = @es.to_compressed_msgpack_stream
       stream = decompress(compressed_stream)
-      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|
         assert_equal @times[i], time
         assert_equal @records[i], record
         i += 1
@@ -490,7 +490,7 @@ module EventTest
       stream = nil
       ensure_data_is_decompressed { stream = @es.to_msgpack_stream }
 
-      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|
         assert_equal @times[i], time
         assert_equal @records[i], record
         i += 1
@@ -505,7 +505,7 @@ module EventTest
       assert_equal @entries, @es.instance_variable_get(:@data)
 
       stream = decompress(compressed_stream)
-      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(stream) { |time, record|
         assert_equal @times[i], time
         assert_equal @records[i], record
         i += 1

data/test/test_fluent_log_event_router.rb

@@ -0,0 +1,99 @@
+require_relative 'helper'
+require 'fluent/fluent_log_event_router'
+require 'fluent/root_agent'
+require 'fluent/system_config'
+
+class FluentLogEventRouterTest < ::Test::Unit::TestCase
+  # @param config [String]
+  def build_config(config)
+    Fluent::Config.parse(config, 'fluent_log_event', '', syntax: :v1)
+  end
+
+  sub_test_case 'NullFluentLogEventRouter does nothing' do
+    test 'emittable? returns false but others does nothing' do
+      null_event_router = Fluent::NullFluentLogEventRouter.new
+      null_event_router.start
+      null_event_router.stop
+      null_event_router.graceful_stop
+      null_event_router.emit_event(nil)
+      assert_false null_event_router.emittable?
+    end
+  end
+
+  sub_test_case '#build' do
+    test 'NullFluentLogEventRouter if root_agent have not internal logger' do
+      root_agent = Fluent::RootAgent.new(log: $log, system_config: Fluent::SystemConfig.new)
+      root_agent.configure(build_config(''))
+
+      d = Fluent::FluentLogEventRouter.build(root_agent)
+      assert_equal Fluent::NullFluentLogEventRouter, d.class
+    end
+
+    test 'FluentLogEventRouter if <match fluent.*> exists in config' do
+      root_agent = Fluent::RootAgent.new(log: $log, system_config: Fluent::SystemConfig.new)
+      root_agent.configure(build_config(<<-CONFIG))
+        <match fluent.*>
+          @type null
+        </match>
+      CONFIG
+
+      d = Fluent::FluentLogEventRouter.build(root_agent)
+      assert_equal Fluent::FluentLogEventRouter, d.class
+    end
+
+    test 'FluentLogEventRouter if <label @FLUENT_LOG> exists in config' do
+      root_agent = Fluent::RootAgent.new(log: $log, system_config: Fluent::SystemConfig.new)
+      root_agent.configure(build_config(<<-CONFIG))
+       <label @FLUENT_LOG>
+         <match *>
+           @type null
+         </match>
+        </label>
+      CONFIG
+
+      d = Fluent::FluentLogEventRouter.build(root_agent)
+      assert_equal Fluent::FluentLogEventRouter, d.class
+    end
+  end
+
+  test 'when calling graceful_stop, it flushes all events' do
+    event_router = []
+    stub(event_router).emit do |tag, time, record|
+      event_router.push([tag, time, record])
+    end
+
+    d = Fluent::FluentLogEventRouter.new(event_router)
+
+    t = Time.now
+    msg = ['tag', t, { 'key' => 'value' }]
+    d.emit_event(msg)
+    d.graceful_stop
+    d.emit_event(msg)
+    d.start
+
+    d.graceful_stop # to call join
+    assert_equal 2, event_router.size
+    assert_equal msg, event_router[0]
+    assert_equal msg, event_router[1]
+  end
+
+  test 'when calling stop, it ignores existing events' do
+    event_router = []
+    stub(event_router).emit do |tag, time, record|
+      event_router.push([tag, time, record])
+    end
+
+    d = Fluent::FluentLogEventRouter.new(event_router)
+
+    t = Time.now
+    msg = ['tag', t, { 'key' => 'value' }]
+    d.emit_event(msg)
+    d.stop
+    d.emit_event(msg)
+    d.start
+
+    d.stop # to call join
+    assert_equal 1, event_router.size
+    assert_equal msg, event_router[0]
+  end
+end

data/test/test_logger_initializer.rb

@@ -0,0 +1,26 @@
+require_relative 'helper'
+require 'fluent/supervisor'
+require 'fileutils'
+
+class LoggerInitializerTest < ::Test::Unit::TestCase
+  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/tmp/logger_initializer#{ENV['TEST_ENV_NUMBER']}")
+
+  teardown do
+    begin
+      FileUtils.rm_rf(TMP_DIR)
+    rescue => _
+    end
+  end
+
+  test 'when path is given' do
+    path = File.join(TMP_DIR, 'fluent_with_path.log')
+
+    assert_false File.exist?(TMP_DIR)
+    logger = Fluent::Supervisor::LoggerInitializer.new(path, Fluent::Log::LEVEL_DEBUG, nil, nil, {})
+
+    assert_nothing_raised do
+      logger.init(:supervisor, 0)
+    end
+    assert_true File.exist?(TMP_DIR)
+  end
+end

data/test/test_supervisor.rb

@@ -30,14 +30,6 @@ class SupervisorTest < ::Test::Unit::TestCase
     File.open(path, "w") {|f| f.write data }
   end
 
-  def test_initialize
-    opts = Fluent::Supervisor.default_options
-    sv = Fluent::Supervisor.new(opts)
-    opts.each { |k, v|
-      assert_equal v, sv.instance_variable_get("@#{k}")
-    }
-  end
-
   def test_read_config
     create_info_dummy_logger
 
@@ -61,10 +53,8 @@ class SupervisorTest < ::Test::Unit::TestCase
 
     sv.instance_variable_set(:@config_path, tmp_dir)
     sv.instance_variable_set(:@use_v1_config, use_v1_config)
-    sv.send(:read_config)
-
-    conf = sv.instance_variable_get(:@conf)
 
+    conf = sv.__send__(:read_config)
     elem = conf.elements.find { |e| e.name == 'source' }
     assert_equal "forward", elem['@type']
     assert_equal "forward_input", elem['@id']
@@ -108,9 +98,8 @@ class SupervisorTest < ::Test::Unit::TestCase
 
     sv.instance_variable_set(:@config_path, tmp_path)
     sv.instance_variable_set(:@use_v1_config, use_v1_config)
-    sv.send(:read_config)
 
-    conf = sv.instance_variable_get(:@conf)
+    conf = sv.__send__(:read_config)
     label = conf.elements.detect {|e| e.name == "label" }
     filter = label.elements.detect {|e| e.name == "filter" }
     record_transformer = filter.elements.detect {|e| e.name = "record_transformer" }
@@ -148,9 +137,7 @@ class SupervisorTest < ::Test::Unit::TestCase
 </system>
     EOC
     conf = Fluent::Config.parse(conf_data, "(test)", "(test_dir)", true)
-    sv.instance_variable_set(:@conf, conf)
-    sv.send(:set_system_config)
-    sys_conf = sv.instance_variable_get(:@system_config)
+    sys_conf = sv.__send__(:build_system_config, conf)
 
     assert_equal '127.0.0.1:24445', sys_conf.rpc_endpoint
     assert_equal true, sys_conf.suppress_repeated_stacktrace
@@ -229,9 +216,7 @@ class SupervisorTest < ::Test::Unit::TestCase
   </system>
     EOC
     conf = Fluent::Config.parse(conf_data, "(test)", "(test_dir)", true)
-    sv.instance_variable_set(:@conf, conf)
-    sv.send(:set_system_config)
-    sys_conf = sv.instance_variable_get(:@system_config)
+    sys_conf = sv.__send__(:build_system_config, conf)
 
     server = DummyServer.new
     server.rpc_endpoint = sys_conf.rpc_endpoint
@@ -387,46 +372,6 @@ class SupervisorTest < ::Test::Unit::TestCase
     assert_equal Fluent::Log::LEVEL_INFO, se_config[:log_level]
   end
 
-  def test_load_config_with_multibyte_string
-    tmp_path = "#{TMP_DIR}/dir/test_multibyte_config.conf"
-    conf_str = %[
-<source>
-  @type forward
-  @id forward_input
-  @label @INPUT
-</source>
-<label @INPUT>
-  <filter>
-    @type record_transformer
-    <record>
-      message こんにちは. ${record["name"]} has made a order of ${record["item"]} just now.
-    </record>
-  </filter>
-  <match>
-    @type stdout
-  </match>
-</label>
-]
-    FileUtils.mkdir_p(File.dirname(tmp_path))
-    File.open(tmp_path, "w:utf-8") {|file| file.write(conf_str) }
-
-    params = {}
-    params['workers'] = 1
-    params['use_v1_config'] = true
-    params['log_path'] = 'test/tmp/supervisor/log'
-    params['suppress_repeated_stacktrace'] = true
-    params['log_level'] = Fluent::Log::LEVEL_INFO
-    params['conf_encoding'] = 'utf-8'
-    load_config_proc =  Proc.new { Fluent::Supervisor.load_config(tmp_path, params) }
-
-    se_config = load_config_proc.call
-    conf = se_config[:fluentd_conf]
-    label = conf.elements.detect {|e| e.name == "label" }
-    filter = label.elements.detect {|e| e.name == "filter" }
-    record_transformer = filter.elements.detect {|e| e.name = "record_transformer" }
-    assert_equal(Encoding::UTF_8, record_transformer["message"].encoding)
-  end
-
   def test_logger
     opts = Fluent::Supervisor.default_options
     sv = Fluent::Supervisor.new(opts)
@@ -463,6 +408,32 @@ class SupervisorTest < ::Test::Unit::TestCase
     assert_equal 10, $log.out.instance_variable_get(:@shift_size)
   end
 
+  def test_inline_config
+    opts = Fluent::Supervisor.default_options
+    opts[:inline_config] = '-'
+    sv = Fluent::Supervisor.new(opts)
+    assert_equal '-', sv.instance_variable_get(:@inline_config)
+
+    inline_config = '<match *>\n@type stdout\n</match>'
+    stub(STDIN).read { inline_config }
+    stub(sv).read_config        # to skip
+    stub(sv).build_system_config { Fluent::SystemConfig.new } # to skip
+
+    sv.configure
+    assert_equal inline_config, sv.instance_variable_get(:@inline_config)
+  end
+
+  def test_log_level_affects
+    opts = Fluent::Supervisor.default_options
+    sv = Fluent::Supervisor.new(opts)
+
+    c = Fluent::Config::Element.new('system', '', { 'log_level' => 'error' }, [])
+    stub(sv).read_config { config_element('ROOT', '', {}, [c]) }
+
+    sv.configure
+    assert_equal Fluent::Log::LEVEL_ERROR, $log.level
+  end
+
   def create_debug_dummy_logger
     dl_opts = {}
     dl_opts[:log_level] = ServerEngine::DaemonLogger::DEBUG