fluentd 1.7.4-x64-mingw32 → 1.8.0-x64-mingw32

data/test/plugin/test_out_forward.rb

@@ -39,7 +39,7 @@ class ForwardOutputTest < Test::Unit::TestCase
 
   def create_driver(conf=CONFIG)
     Fluent::Test::Driver::Output.new(Fluent::Plugin::ForwardOutput) {
-      attr_reader :sent_chunk_ids, :ack_handler
+      attr_reader :sent_chunk_ids, :ack_handler, :discovery_manager
 
       def initialize
         super
@@ -260,6 +260,30 @@ EOL
     end
   end
 
+  test 'server is an abbreviation of static type of service_discovery' do
+    @d = d = create_driver(%[
+<server>
+  host 127.0.0.1
+  port 1234
+</server>
+
+<service_discovery>
+  @type static
+
+  <service>
+    host 127.0.0.1
+    port 1235
+  </service>
+</service_discovery>
+    ])
+
+    assert_equal 2, d.instance.discovery_manager.services.size
+    assert_equal '127.0.0.1', d.instance.discovery_manager.services[0].host
+    assert_equal 1234, d.instance.discovery_manager.services[0].port
+    assert_equal '127.0.0.1', d.instance.discovery_manager.services[1].host
+    assert_equal 1235, d.instance.discovery_manager.services[1].port
+  end
+
   test 'compress_default_value' do
     @d = d = create_driver
     assert_equal :text, d.instance.compress
@@ -945,6 +969,26 @@ EOL
     end
   end
 
+  test 'when out_forward has @id' do
+    # cancel https://github.com/fluent/fluentd/blob/077508ac817b7637307434d0c978d7cdc3d1c534/lib/fluent/plugin_id.rb#L43-L53
+    # it always return true in test
+    mock.proxy(Fluent::Plugin).new_sd(:static, anything) { |v|
+      stub(v).plugin_id_for_test? { false }
+    }.once
+
+    output = Fluent::Test::Driver::Output.new(Fluent::Plugin::ForwardOutput) {
+      def plugin_id_for_test?
+        false
+      end
+    }
+
+    assert_nothing_raised do
+      output.configure(CONFIG + %[
+        @id unique_out_forward
+      ])
+    end
+  end
+
   sub_test_case 'verify_connection_at_startup' do
     test 'nodes are not available' do
       @d = d = create_driver(CONFIG + %[

data/test/plugin/test_out_stdout.rb

@@ -37,6 +37,25 @@ class StdoutOutputTest < Test::Unit::TestCase
       end
     end
 
+    test 'configure with time_format' do
+      d = create_driver(CONFIG + <<-CONF)
+        <format>
+          @type stdout
+          time_format %Y-%m-%dT%H:%M:%S.%L%z
+        </format>
+      CONF
+
+      time = event_time
+      out = capture_log do
+        d.run(default_tag: 'test') do
+          d.feed(time, {'test' => 'test'})
+        end
+      end
+
+      t = Time.at(time).localtime.strftime("%Y-%m-%dT%H:%M:%S.%L%z")
+      assert_equal "#{t} test: {\"test\":\"test\"}\n", out
+    end
+
     test 'emit with default configuration' do
       d = create_driver
       time = event_time()
@@ -157,6 +176,23 @@ class StdoutOutputTest < Test::Unit::TestCase
     end
   end
 
+  data(
+    'utc and !localtime' => "utc true\nlocaltime false",
+    '!utc and localtime' => "utc false\nlocaltime true")
+  test 'success when configure with localtime and utc' do |c|
+    assert_nothing_raised do
+      create_driver(CONFIG + c)
+    end
+  end
+
+  data('utc and localtime' => "utc true\nlocaltime true",
+       '!utc and !localtime' => "utc false\nlocaltime false")
+  test 'raise an error when configure with localtime and utc' do |c|
+    assert_raise(Fluent::ConfigError.new('both of utc and localtime are specified, use only one of them')) do
+      create_driver(CONFIG + c)
+    end
+  end
+
   # Capture the log output of the block given
   def capture_log(&block)
     tmp = $log
@@ -167,4 +203,3 @@ class StdoutOutputTest < Test::Unit::TestCase
     $log = tmp
   end
 end
-

data/test/plugin/test_out_stream.rb

@@ -31,10 +31,10 @@ module StreamOutputTest
     d.emit({"a"=>2}, time)
 
     expect = ["test",
-        Fluent::Engine.msgpack_factory.packer.write([time,{"a"=>1}]).to_s +
-        Fluent::Engine.msgpack_factory.packer.write([time,{"a"=>2}]).to_s
+        Fluent::MessagePackFactory.msgpack_packer.write([time,{"a"=>1}]).to_s +
+        Fluent::MessagePackFactory.msgpack_packer.write([time,{"a"=>2}]).to_s
       ]
-    expect = Fluent::Engine.msgpack_factory.packer.write(expect).to_s
+    expect = Fluent::MessagePackFactory.msgpack_packer.write(expect).to_s
 
     result = d.run
     assert_equal(expect, result)

data/test/plugin/test_output.rb

@@ -378,6 +378,30 @@ class OutputTest < Test::Unit::TestCase
       assert { logs.any? { |log| log.include?("${chunk_id} is not allowed in this plugin") } }
     end
 
+    test '#extract_placeholders logs warn message with not replaced key' do
+      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
+      tmpl = "/mypath/${key1}/test"
+      t = event_time('2016-04-11 20:30:00 +0900')
+      v = { key1: "value1" }
+      m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
+      @i.extract_placeholders(tmpl, m)
+      logs = @i.log.out.logs
+
+      assert { logs.any? { |log| log.include?("chunk key placeholder 'key1' not replaced. template:#{tmpl}") } }
+    end
+
+    test '#extract_placeholders logs warn message with not replaced key if variables exist and chunk_key is not empty' do
+      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'key1')]))
+      tmpl = "/mypath/${key1}/${key2}/test"
+      t = event_time('2016-04-11 20:30:00 +0900')
+      v = { key1: "value1" }
+      m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
+      @i.extract_placeholders(tmpl, m)
+      logs = @i.log.out.logs
+
+      assert { logs.any? { |log| log.include?("chunk key placeholder 'key2' not replaced. template:#{tmpl}") } }
+    end
+
     sub_test_case '#placeholder_validators' do
       test 'returns validators for time, tag and keys when a template has placeholders even if plugin is not configured with these keys' do
         @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
@@ -870,7 +894,7 @@ class OutputTest < Test::Unit::TestCase
 
   test 'raises an error if timekey is less than equal 0' do
     i = create_output(:delayed)
-    assert_raise Fluent::ConfigError.new('timekey should be greater than 0. current timekey: 0.0') do
+    assert_raise Fluent::ConfigError.new("<buffer ...> argument includes 'time', but timekey is not configured") do
       i.configure(config_element('ROOT','',{},[config_element('buffer', 'time', { "timekey" => nil })]))
     end
 

data/test/plugin/test_sd_file.rb

@@ -0,0 +1,211 @@
+require_relative '../helper'
+require 'fluent/plugin/sd_file'
+require 'fileutils'
+require 'json'
+
+class FileServiceDiscoveryTest < ::Test::Unit::TestCase
+  setup do
+    @dir = File.expand_path('data/sd_file', __dir__)
+    FileUtils.mkdir_p(File.join(@dir, 'tmp'))
+  end
+
+  teardown do
+    FileUtils.rm_r(File.join(@dir, 'tmp'))
+  end
+
+  sub_test_case 'configure' do
+    test 'load yml' do
+      sdf = Fluent::Plugin::FileServiceDiscovery.new
+      sdf.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'config.yml') }))
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:file, '127.0.0.1', 24224, 'test1', 1, false, 'user1', 'pass1', 'key1'), sdf.services[0]
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:file, '127.0.0.1', 24225, nil, 1), sdf.services[1]
+    end
+
+    test 'load yaml' do
+      sdf = Fluent::Plugin::FileServiceDiscovery.new
+      sdf.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'config.yaml') }))
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:file, '127.0.0.1', 24224, 'test1', 1, false, 'user1', 'pass1', 'key1'), sdf.services[0]
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:file, '127.0.0.1', 24225, nil, 1), sdf.services[1]
+    end
+
+    test 'load json' do
+      sdf = Fluent::Plugin::FileServiceDiscovery.new
+      sdf.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'config.json') }))
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:file, '127.0.0.1', 24224, 'test1', 1, false, 'user1', 'pass1', 'key1'), sdf.services[0]
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:file, '127.0.0.1', 24225, nil, 1), sdf.services[1]
+    end
+
+    test 'regard as yaml if ext is not givened' do
+      sdf = Fluent::Plugin::FileServiceDiscovery.new
+      sdf.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'config') }))
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:file, '127.0.0.1', 24224, 'test1', 1, false, 'user1', 'pass1', 'key1'), sdf.services[0]
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:file, '127.0.0.1', 24225, nil, 1), sdf.services[1]
+    end
+
+    test 'raise an error if config has error' do
+      sdf = Fluent::Plugin::FileServiceDiscovery.new
+      e = assert_raise Fluent::ConfigError do
+        sdf.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'invalid_config.yaml') }))
+      end
+      assert_match(/path=/, e.message)
+    end
+
+    test 'raise an error if config file does not exist' do
+      sdf = Fluent::Plugin::FileServiceDiscovery.new
+      e = assert_raise Fluent::ConfigError do
+        sdf.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'invalid_not_found.json') }))
+      end
+      assert_match(/not found/, e.message)
+    end
+  end
+
+  sub_test_case '#start' do
+    module TestStatEventHelperWrapper
+      # easy to control statsevent
+      def event_loop_attach(watcher)
+        unless watcher.is_a?(Fluent::Plugin::FileServiceDiscovery::StatWatcher)
+          super
+          return
+        end
+
+        @test_stat_event_helper_wrapper_watchers ||= []
+        @test_stat_event_helper_wrapper_watchers << watcher
+
+        @test_stat_event_helper_wrapper_context = Fiber.new do
+          loop do
+            @test_stat_event_helper_wrapper_watchers.each do |w|
+              w.on_change('old', 'new')
+            end
+
+            if Fiber.yield == :finish
+              break
+            end
+          end
+        end
+        resume
+      end
+
+      def resume
+        @test_stat_event_helper_wrapper_context.resume(:resume)
+      end
+
+      def shutdown
+        super
+
+        if @test_stat_event_helper_wrapper_context
+          @test_stat_event_helper_wrapper_context.resume(:finish)
+        end
+      end
+    end
+
+    def create_tmp_config(path, body)
+      File.write(File.join(@dir, 'tmp', path), body)
+    end
+
+    setup do
+      sdf = Fluent::Plugin::FileServiceDiscovery.new
+      @sd_file = sdf
+    end
+
+    teardown do
+      if @sd_file
+        @sd_file.stop unless @sd_file.stopped?
+        @sd_file.before_shutdown unless @sd_file.before_shutdown?
+        @sd_file.shutdown unless @sd_file.shutdown?
+        @sd_file.after_shutdown unless @sd_file.after_shutdown?
+        @sd_file.close unless @sd_file.closed?
+        @sd_file.terminate unless @sd_file.terminated?
+      end
+    end
+
+    test 'Skip if file is not updated' do
+      @sd_file.extend(TestStatEventHelperWrapper)
+
+      create_tmp_config('config.json', JSON.generate([{ port: 1233, host: '127.0.0.1' }]))
+      @sd_file.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'config.yml') }))
+      queue = []
+      mock.proxy(@sd_file).refresh_file(queue).twice
+
+      @sd_file.start(queue)
+      assert_empty queue
+
+      @sd_file.resume
+      assert_empty queue
+    end
+
+    test 'Skip if file is invalid contents' do
+      @sd_file.extend(TestStatEventHelperWrapper)
+
+      create_tmp_config('config.json', JSON.generate([{ port: 1233, host: '127.0.0.1' }]))
+      @sd_file.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'config.yml') }))
+
+      queue = []
+      @sd_file.start(queue)
+
+      mock.proxy(@sd_file).refresh_file(queue).once
+      create_tmp_config('test.json', 'invalid contents')
+      @sd_file.resume
+
+      assert_empty queue
+    end
+
+    test 'if service is updated, service_in and service_out event happen' do
+      @sd_file.extend(TestStatEventHelperWrapper)
+
+      create_tmp_config('test.json', JSON.generate([{ port: 1233, host: '127.0.0.1' }]))
+      @sd_file.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'tmp/test.json') }))
+
+      queue = []
+      @sd_file.start(queue)
+      create_tmp_config('test.json', JSON.generate([{ port: 1234, host: '127.0.0.1' }]))
+      @sd_file.resume
+
+      assert_equal 2, queue.size
+      join = queue.shift
+      drain = queue.shift
+      assert_equal Fluent::Plugin::ServiceDiscovery::SERVICE_IN, join.type
+      assert_equal 1234, join.service.port
+      assert_equal '127.0.0.1', join.service.host
+
+      assert_equal Fluent::Plugin::ServiceDiscovery::SERVICE_OUT, drain.type
+      assert_equal 1233, drain.service.port
+      assert_equal '127.0.0.1', drain.service.host
+    end
+
+    test 'if service is deleted, service_out event happens' do
+      @sd_file.extend(TestStatEventHelperWrapper)
+
+      create_tmp_config('test.json', JSON.generate([{ port: 1233, host: '127.0.0.1' }, { port: 1234, host: '127.0.0.2' }]))
+      @sd_file.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'tmp/test.json') }))
+
+      queue = []
+      @sd_file.start(queue)
+      create_tmp_config('test.json', JSON.generate([{ port: 1233, host: '127.0.0.1' }]))
+      @sd_file.resume
+
+      assert_equal 1, queue.size
+      drain = queue.shift
+      assert_equal Fluent::Plugin::ServiceDiscovery::SERVICE_OUT, drain.type
+      assert_equal 1234, drain.service.port
+      assert_equal '127.0.0.2', drain.service.host
+    end
+
+    test 'if new service is added, service_in event happens' do
+      @sd_file.extend(TestStatEventHelperWrapper)
+
+      create_tmp_config('test.json', JSON.generate([{ port: 1233, host: '127.0.0.1' }]))
+      @sd_file.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'tmp/test.json') }))
+
+      queue = []
+      @sd_file.start(queue)
+      create_tmp_config('test.json', JSON.generate([{ port: 1233, host: '127.0.0.1' }, { port: 1234, host: '127.0.0.2' }]))
+      @sd_file.resume
+
+      assert_equal 1, queue.size
+      join = queue.shift
+      assert_equal Fluent::Plugin::ServiceDiscovery::SERVICE_IN, join.type
+      assert_equal 1234, join.service.port
+      assert_equal '127.0.0.2', join.service.host
+    end
+  end
+end

data/test/plugin_helper/service_discovery/test_manager.rb

@@ -0,0 +1,93 @@
+require_relative '../../helper'
+require 'fluent/plugin_helper/service_discovery/manager'
+
+class TestServiceDiscoveryManager < ::Test::Unit::TestCase
+  setup do
+    @sd_file_dir = File.expand_path('../../plugin/data/sd_file', __dir__)
+  end
+
+  class TestSdPlugin < Fluent::Plugin::ServiceDiscovery
+    Fluent::Plugin.register_sd('test_sd', self)
+
+    def initialize
+      super
+    end
+
+    def service_in(host, port)
+      s = Fluent::Plugin::ServiceDiscovery::Service.new(:sd_test, host, port)
+      @queue << Fluent::Plugin::ServiceDiscovery.service_in_msg(s)
+    end
+
+    def service_out(host, port)
+      s = Fluent::Plugin::ServiceDiscovery::Service.new(:sd_test, host, port)
+      @queue << Fluent::Plugin::ServiceDiscovery.service_out_msg(s)
+    end
+
+    def start(queue)
+      @queue = queue
+
+      super
+    end
+  end
+
+  sub_test_case '#configure' do
+    test 'build sd plugins and services' do
+      sdm = Fluent::PluginHelper::ServiceDiscovery::Manager.new(log: $log)
+      sdm.configure(
+        [
+          { type: :file, conf: config_element('service_discovery', '', { 'path' => File.join(@sd_file_dir, 'config.yml') }) },
+          { type: :static, conf: config_element('root', '', {}, [config_element('service', '', { 'host' => '127.0.0.2', 'port' => '5432' })]) },
+        ],
+      )
+
+      assert_equal 3, sdm.services.size
+      assert_equal 24224, sdm.services[0].port
+      assert_equal '127.0.0.1', sdm.services[0].host
+
+      assert_equal 24225, sdm.services[1].port
+      assert_equal '127.0.0.1', sdm.services[1].host
+
+      assert_equal 5432, sdm.services[2].port
+      assert_equal '127.0.0.2', sdm.services[2].host
+
+      assert_false sdm.static_config?
+    end
+
+    test 'no need to timer if only static' do
+      sdm = Fluent::PluginHelper::ServiceDiscovery::Manager.new(log: $log)
+      sdm.configure(
+        [{ type: :static, conf: config_element('root', '', {}, [config_element('service', '', { 'host' => '127.0.0.2', 'port' => '5432' })]) }]
+      )
+
+      assert_equal 1, sdm.services.size
+      assert_equal 5432, sdm.services[0].port
+      assert_equal '127.0.0.2', sdm.services[0].host
+
+      assert_true sdm.static_config?
+    end
+  end
+
+  sub_test_case '#run_once' do
+    test 'if new service added and deleted' do
+      sdm = Fluent::PluginHelper::ServiceDiscovery::Manager.new(log: $log)
+      t = TestSdPlugin.new
+      mock(Fluent::Plugin).new_sd(:sd_test, anything) { t }
+      sdm.configure([{ type: :sd_test, conf: config_element('service_discovery', '', {})}])
+      sdm.start
+
+      assert_equal 0, sdm.services.size
+
+      t.service_in('127.0.0.1', '1234')
+
+      sdm.run_once
+      assert_equal 1, sdm.services.size
+      assert_equal '127.0.0.1', sdm.services[0].host
+      assert_equal '1234', sdm.services[0].port
+
+      t.service_out('127.0.0.1', '1234')
+
+      sdm.run_once
+      assert_equal 0, sdm.services.size
+    end
+  end
+end