fluentd 1.12.0-x86-mingw32 → 1.13.0-x86-mingw32

data/test/plugin_helper/test_http_server_helper.rb

@@ -132,7 +132,9 @@ class HttpHelperTest < Test::Unit::TestCase
         error = e
       end
 
-      resp = Response.new(response.status.to_s, response.body.read, response.headers)
+      if response
+        resp = Response.new(response.status.to_s, response.body.read, response.headers)
+      end
     end
 
     if error
@@ -352,7 +354,7 @@ class HttpHelperTest < Test::Unit::TestCase
           watcher.should_receive(:stop).once
         end
 
-        stub(Fluent::PluginHelper::HttpServer::Server).new(anything) { server }
+        stub(Fluent::PluginHelper::HttpServer::Server).new(addr: anything, port: anything, logger: anything, default_app: anything) { server }
         driver.http_server_create_http_server(:http_server_helper_test, addr: '127.0.0.1', port: PORT, logger: NULL_LOGGER) do
           # nothing
         end

data/test/plugin_helper/test_server.rb

@@ -234,7 +234,13 @@ class ServerPluginHelperTest < Test::Unit::TestCase
       assert_raise(ArgumentError.new("BUG: backlog is available for tcp/tls")) do
         @d.__send__(m, :myserver, PORT, proto: proto, backlog: 500){|x| x }
       end
-      assert_raise(ArgumentError.new("BUG: send_keepalive_packet is available for tcp")) do
+    end
+
+    data(
+      'server_create udp' => [:server_create, :udp],
+    )
+    test 'raise error if tcp/tls send_keepalive_packet option is specified for udp' do |(m, proto)|
+      assert_raise(ArgumentError.new("BUG: send_keepalive_packet is available for tcp/tls")) do
         @d.__send__(m, :myserver, PORT, proto: proto, send_keepalive_packet: true){|x| x }
       end
     end
@@ -832,8 +838,9 @@ class ServerPluginHelperTest < Test::Unit::TestCase
     chain_cert.sign(root_key, "sha256")
 
     server_cert, server_key, _ = CertUtil.cert_option_generate_pair(create_server_options, chain_cert.subject)
-    server_cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(false)]))
-    server_cert.add_extension OpenSSL::X509::Extension.new('nsCertType', 'server')
+    factory = OpenSSL::X509::ExtensionFactory.new
+    server_cert.add_extension(factory.create_extension('basicConstraints', 'CA:FALSE'))
+    server_cert.add_extension(factory.create_extension('nsCertType', 'server'))
     server_cert.sign(chain_key, "sha256")
 
     # write chained cert
@@ -1299,7 +1306,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
 
     test 'can accept all keyword arguments valid for tcp/tls server' do
       assert_nothing_raised do
-        @d.server_create_tls(:s, PORT, bind: '127.0.0.1', shared: false, resolve_name: true, linger_timeout: 10, backlog: 500, tls_options: @tls_options) do |data, conn|
+        @d.server_create_tls(:s, PORT, bind: '127.0.0.1', shared: false, resolve_name: true, linger_timeout: 10, backlog: 500, tls_options: @tls_options, send_keepalive_packet: true) do |data, conn|
           # ...
         end
       end
@@ -1494,8 +1501,13 @@ class ServerPluginHelperTest < Test::Unit::TestCase
       test "can't connect with different TLS version" do
         @d.server_create_tls(:s, PORT, tls_options: @tls_options) do |data, conn|
         end
+        if defined?(OpenSSL::SSL::TLS1_3_VERSION)
+          version = :'TLS1_3'
+        else
+          version = :'TLS1_1'
+        end
         assert_raise(OpenSSL::SSL::SSLError, Errno::ECONNRESET) {
-          open_tls_session('127.0.0.1', PORT, cert_path: @cert_path, version: :'TLS1_1') do |sock|
+          open_tls_session('127.0.0.1', PORT, cert_path: @cert_path, version: version) do |sock|
           end
         }
       end
@@ -1503,14 +1515,21 @@ class ServerPluginHelperTest < Test::Unit::TestCase
       test "can specify multiple TLS versions by min_version/max_version" do
         omit "min_version=/max_version= is not supported" unless Fluent::TLS::MIN_MAX_AVAILABLE
 
-        opts = @tls_options.merge(min_version: :'TLS1_1', max_version: :'TLSv1_2')
+        min_version = :'TLS1_2'
+        if defined?(OpenSSL::SSL::TLS1_3_VERSION)
+          max_version = :'TLS1_3'
+        else
+          max_version = :'TLS1_2'
+        end
+
+        opts = @tls_options.merge(min_version: min_version, max_version: max_version)
         @d.server_create_tls(:s, PORT, tls_options: opts) do |data, conn|
         end
         assert_raise(OpenSSL::SSL::SSLError, Errno::ECONNRESET) {
           open_tls_session('127.0.0.1', PORT, cert_path: @cert_path, version: :'TLS1') do |sock|
           end
         }
-        [:'TLS1_1', :'TLS1_2'].each { |ver|
+        [min_version, max_version].each { |ver|
           assert_nothing_raised {
             open_tls_session('127.0.0.1', PORT, cert_path: @cert_path, version: ver) do |sock|
             end

data/test/plugin_helper/test_service_discovery.rb

@@ -17,6 +17,24 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
     end
   end
 
+  class DummyPlugin < Fluent::Plugin::TestBase
+    helpers :service_discovery
+
+    def configure(conf)
+      super
+      service_discovery_configure(:service_discovery_helper_test, static_default_service_directive: 'node')
+    end
+
+    def select_service(&block)
+      service_discovery_select_service(&block)
+    end
+
+    # Make these mehtod public
+    def discovery_manager
+      super
+    end
+  end
+
   setup do
     @sd_file_dir = File.expand_path('../plugin/data/sd_file', __dir__)
 
@@ -33,7 +51,7 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
     end
   end
 
-  test 'start discovery manager' do
+  test 'support calling #service_discovery_create_manager and #discovery_manager from plugin' do
     d = @d = Dummy.new
 
     d.service_discovery_create_manager(
@@ -55,13 +73,30 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
     assert_equal 1234, services[0].port
   end
 
-  test 'call timer_execute if dynamic configuration' do
-    d = @d = Dummy.new
+  test 'start discovery manager' do
+    d = @d = DummyPlugin.new
 
-    d.service_discovery_create_manager(
-      :service_discovery_helper_test,
-      configurations: [{ type: :file, conf: config_element('file_config', '', { 'path' => File.join(@sd_file_dir, 'config.yml') }) }],
-    )
+    services = [config_element('service', '', { 'host' => '127.0.0.1', 'port' => '1234' })]
+    d.configure(config_element('root', '', {}, [config_element('service_discovery', '', {'@type' => 'static'}, services)]))
+
+    assert_true !!d.discovery_manager
+
+    mock.proxy(d.discovery_manager).start.once
+    mock.proxy(d).timer_execute(:service_discovery_helper_test, anything).never
+
+    d.start
+    d.event_loop_wait_until_start
+
+    assert_equal 1, d.discovery_manager.services.size
+    d.select_service do |serv|
+      assert_equal "127.0.0.1", serv.host
+      assert_equal 1234, serv.port
+    end
+  end
+
+  test 'call timer_execute if dynamic configuration' do
+    d = @d = DummyPlugin.new
+    d.configure(config_element('root', '', {}, [config_element('service_discovery', '', { '@type' => 'file', 'path' => File.join(@sd_file_dir, 'config.yml' )})]))
 
     assert_true !!d.discovery_manager
     mock.proxy(d.discovery_manager).start.once
@@ -71,25 +106,22 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
   end
 
   test 'exits service discovery instances without any errors' do
-    d = @d = Dummy.new
+    d = @d = DummyPlugin.new
     mockv = flexmock('dns_resolver', getaddress: '127.0.0.1')
               .should_receive(:getresources)
               .and_return([Resolv::DNS::Resource::IN::SRV.new(1, 10, 8081, 'service1.example.com')])
               .mock
     mock(Resolv::DNS).new { mockv }
 
-    d.service_discovery_create_manager(
-      :service_discovery_helper_test2,
-      configurations: [{ type: :srv, conf: config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com' }) }],
-    )
+    d.configure(config_element('root', '', {}, [config_element('service_discovery', '', { '@type' => 'srv', 'service' => 'service1', 'hostname' => 'example.com' })]))
 
     assert_true !!d.discovery_manager
     mock.proxy(d.discovery_manager).start.once
-    mock(d).timer_execute(:service_discovery_helper_test2, anything).once
+    mock(d).timer_execute(:service_discovery_helper_test, anything).once
 
     # To avoid claring `@logs` during `terminate` step
     # https://github.com/fluent/fluentd/blob/bc78d889f93dad8c2a4e0ad1ca802546185dacba/lib/fluent/test/log.rb#L33
-    mock(d.log).reset.twice
+    mock(d.log).reset.times(3)
 
     d.start
     d.event_loop_wait_until_start
@@ -102,4 +134,32 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
 
     assert_false(d.log.out.logs.any? { |e| e.match?(/thread doesn't exit correctly/) })
   end
+
+  test 'static service discovery will be configured automatically when default service directive is specified' do
+    d = @d = DummyPlugin.new
+
+    nodes = [
+      config_element('node', '', { 'host' => '192.168.0.1', 'port' => '24224' }),
+      config_element('node', '', { 'host' => '192.168.0.2', 'port' => '24224' })
+    ]
+    d.configure(config_element('root', '', {}, nodes))
+
+    assert_true !!d.discovery_manager
+
+    mock.proxy(d.discovery_manager).start.once
+    mock.proxy(d).timer_execute(:service_discovery_helper_test, anything).never
+
+    d.start
+    d.event_loop_wait_until_start
+
+    assert_equal 2, d.discovery_manager.services.size
+    d.select_service do |serv|
+      assert_equal "192.168.0.1", serv.host
+      assert_equal 24224, serv.port
+    end
+    d.select_service do |serv|
+      assert_equal "192.168.0.2", serv.host
+      assert_equal 24224, serv.port
+    end
+  end
 end

data/test/test_config.rb

@@ -160,7 +160,8 @@ class ConfigTest < Test::Unit::TestCase
     prepare_config
     opts = {
       :config_path => "#{TMP_DIR}/config_test_1.conf",
-      :inline_config => "<source>\n  type http\n  port 2222\n </source>"
+      :inline_config => "<source>\n  type http\n  port 2222\n </source>",
+      :use_v1_config => false
     }
     assert_nothing_raised do
       Fluent::Supervisor.new(opts)

data/test/test_event.rb

@@ -401,6 +401,22 @@ module EventTest
         i += 1
       }
     end
+
+    # `any?` represents an Enumerable method which calls `each` internally
+    test 'size_after_any' do
+      @es.any?
+
+      assert_equal 2, @es.size
+    end
+
+    # `any?` represents an Enumerable method which calls `each` internally
+    test 'each_after_any' do
+      @es.any?
+
+      count = 0
+      @es.each { |time, record| count += 1 }
+      assert_equal 2, count
+    end
   end
 
   class CompressedMessagePackEventStreamTest < ::Test::Unit::TestCase

data/test/test_formatter.rb

@@ -184,6 +184,36 @@ module FormatterTest
 
       assert_equal("message=awesome,greeting=hello#{@newline}", formatted)
     end
+
+    def record_with_tab
+      {'message' => "awe\tsome", 'greeting' => "hello\t"}
+    end
+
+    def test_format_suppresses_tab
+      @formatter.configure({})
+      formatted = @formatter.format(tag, @time, record_with_tab)
+
+      assert_equal("message:awe some\tgreeting:hello #{@newline}", formatted)
+    end
+
+    def test_format_suppresses_tab_custom_replacement
+      @formatter.configure(
+        'replacement'      => 'X',
+      )
+      formatted = @formatter.format(tag, @time, record_with_tab)
+
+      assert_equal("message:aweXsome\tgreeting:helloX#{@newline}", formatted)
+    end
+
+    def test_format_suppresses_custom_delimiter
+      @formatter.configure(
+        'delimiter'       => 'w',
+        'label_delimiter' => '=',
+      )
+      formatted = @formatter.format(tag, @time, record)
+
+      assert_equal("message=a esomewgreeting=hello#{@newline}", formatted)
+    end
   end
 
   class CsvFormatterTest < ::Test::Unit::TestCase

data/test/test_output.rb

@@ -117,7 +117,7 @@ module FluentOutputTest
       end
 
       mock(d.instance.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
-                                { primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output" })
+                                primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output")
       d.configure(CONFIG + %[
         <secondary>
           type test2
@@ -133,7 +133,7 @@ module FluentOutputTest
       d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::ObjectBufferedOutput)
 
       mock(d.instance.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
-                                { primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output" }).never
+                                primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output").never
       d.configure(CONFIG + %[
         <secondary>
           type test2

data/test/test_supervisor.rb

@@ -7,6 +7,7 @@ require_relative 'test_plugin_classes'
 require 'net/http'
 require 'uri'
 require 'fileutils'
+require 'tempfile'
 
 if Fluent.windows?
   require 'win32/event'
@@ -489,6 +490,40 @@ class SupervisorTest < ::Test::Unit::TestCase
     assert_equal 10, $log.out.instance_variable_get(:@shift_size)
   end
 
+  sub_test_case "system log rotation" do
+    def parse_text(text)
+      basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
+      Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }
+    end
+
+    def test_override_default_log_rotate
+      Tempfile.open do |file|
+        config = parse_text(<<-EOS)
+          <system>
+            <log>
+              rotate_age 3
+              rotate_size 300
+            </log>
+          </system>
+        EOS
+        file.puts(config)
+        file.flush
+        opts = Fluent::Supervisor.default_options.merge(
+          log_path: "#{TMP_DIR}/test.log", config_path: file.path
+        )
+        sv = Fluent::Supervisor.new(opts)
+
+        log = sv.instance_variable_get(:@log)
+        log.init(:standalone, 0)
+        logger = $log.instance_variable_get(:@logger)
+
+        assert_equal([3, 300],
+                     [logger.instance_variable_get(:@rotate_age),
+                      logger.instance_variable_get(:@rotate_size)])
+      end
+    end
+  end
+
   def test_inline_config
     omit 'this feature is deprecated. see https://github.com/fluent/fluentd/issues/2711'
 
@@ -517,6 +552,37 @@ class SupervisorTest < ::Test::Unit::TestCase
     assert_equal Fluent::Log::LEVEL_ERROR, $log.level
   end
 
+  def test_enable_shared_socket
+    server = DummyServer.new
+    begin
+      ENV.delete('SERVERENGINE_SOCKETMANAGER_PATH')
+      server.before_run
+      sleep 0.1 if Fluent.windows? # Wait for starting windows event thread
+      assert_not_nil(ENV['SERVERENGINE_SOCKETMANAGER_PATH'])
+    ensure
+      server.after_run
+      ENV.delete('SERVERENGINE_SOCKETMANAGER_PATH')
+    end
+  end
+
+  def test_disable_shared_socket
+    server = DummyServer.new
+    def server.config
+      {
+        :disable_shared_socket => true,
+      }
+    end
+    begin
+      ENV.delete('SERVERENGINE_SOCKETMANAGER_PATH')
+      server.before_run
+      sleep 0.1 if Fluent.windows? # Wait for starting windows event thread
+      assert_nil(ENV['SERVERENGINE_SOCKETMANAGER_PATH'])
+    ensure
+      server.after_run
+      ENV.delete('SERVERENGINE_SOCKETMANAGER_PATH')
+    end
+  end
+
   def create_debug_dummy_logger
     dl_opts = {}
     dl_opts[:log_level] = ServerEngine::DaemonLogger::DEBUG

data/test/test_time_parser.rb

@@ -228,4 +228,113 @@ class TimeParserTest < ::Test::Unit::TestCase
       assert_equal_event_time(time, parser.parse("#{time.sec}.#{time.nsec}"))
     end
   end
+
+  sub_test_case 'MixedTimeParser fallback' do
+    class DummyForTimeParser
+      include Fluent::Configurable
+      include Fluent::TimeMixin::Parser
+    end
+
+    test 'no time_format_fallbacks failure' do
+      i = DummyForTimeParser.new
+      assert_raise(Fluent::ConfigError.new("time_type is :mixed but time_format and time_format_fallbacks is empty.")) do
+        i.configure(config_element('parse', '', {'time_type' => 'mixed'}))
+      end
+    end
+
+    test 'fallback time format failure' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format_fallbacks' => ['%iso8601']}))
+      parser = i.time_parser_create
+      assert_raise(Fluent::TimeParser::TimeParseError.new("invalid time format: value = INVALID, even though fallbacks: Fluent::TimeParser")) do
+        parser.parse("INVALID")
+      end
+    end
+
+    test 'primary format is unixtime, secondary %iso8601 is used' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => 'unixtime',
+                                  'time_format_fallbacks' => ['%iso8601']}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(time, parser.parse('2021-01-01T12:00:00+0900'))
+    end
+
+    test 'primary format is %iso8601, secondary unixtime is used' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => '%iso8601',
+                                  'time_format_fallbacks' => ['unixtime']}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(time, parser.parse("#{time.sec}"))
+    end
+
+    test 'primary format is %iso8601, no secondary is used' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => '%iso8601'}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(time, parser.parse("2021-01-01T12:00:00+0900"))
+    end
+
+    test 'primary format is unixtime, no secondary is used' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => 'unixtime'}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(time, parser.parse("#{time.sec}"))
+    end
+
+    test 'primary format is %iso8601, raise error because of no appropriate secondary' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => '%iso8601'}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_raise("Fluent::TimeParser::TimeParseError: invalid time format: value = #{time.sec}, even though fallbacks: Fluent::TimeParser") do
+        parser.parse("#{time.sec}")
+      end
+    end
+
+    test 'primary format is unixtime, raise error because of no appropriate secondary' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => 'unixtime'}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_raise("Fluent::TimeParser::TimeParseError: invalid time format: value = #{time}, even though fallbacks: Fluent::NumericTimeParser") do
+        parser.parse("2021-01-01T12:00:00+0900")
+      end
+    end
+
+    test 'fallback to unixtime' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '', {'time_type' => 'mixed',
+                                               'time_format_fallbacks' => ['%iso8601', 'unixtime']}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(Fluent::EventTime.new(time.to_i), parser.parse("#{time.sec}"))
+    end
+
+    test 'fallback to %iso8601' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '', {'time_type' => 'mixed',
+                                               'time_format_fallbacks' => ['unixtime', '%iso8601']}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(time, parser.parse('2021-01-01T12:00:00+0900'))
+    end
+  end
 end