fluentd 1.12.1 → 1.13.1

data/test/plugin_helper/test_service_discovery.rb

@@ -17,6 +17,24 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
     end
   end
 
+  class DummyPlugin < Fluent::Plugin::TestBase
+    helpers :service_discovery
+
+    def configure(conf)
+      super
+      service_discovery_configure(:service_discovery_helper_test, static_default_service_directive: 'node')
+    end
+
+    def select_service(&block)
+      service_discovery_select_service(&block)
+    end
+
+    # Make these mehtod public
+    def discovery_manager
+      super
+    end
+  end
+
   setup do
     @sd_file_dir = File.expand_path('../plugin/data/sd_file', __dir__)
 
@@ -33,7 +51,7 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
     end
   end
 
-  test 'start discovery manager' do
+  test 'support calling #service_discovery_create_manager and #discovery_manager from plugin' do
     d = @d = Dummy.new
 
     d.service_discovery_create_manager(
@@ -55,13 +73,30 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
     assert_equal 1234, services[0].port
   end
 
-  test 'call timer_execute if dynamic configuration' do
-    d = @d = Dummy.new
+  test 'start discovery manager' do
+    d = @d = DummyPlugin.new
 
-    d.service_discovery_create_manager(
-      :service_discovery_helper_test,
-      configurations: [{ type: :file, conf: config_element('file_config', '', { 'path' => File.join(@sd_file_dir, 'config.yml') }) }],
-    )
+    services = [config_element('service', '', { 'host' => '127.0.0.1', 'port' => '1234' })]
+    d.configure(config_element('root', '', {}, [config_element('service_discovery', '', {'@type' => 'static'}, services)]))
+
+    assert_true !!d.discovery_manager
+
+    mock.proxy(d.discovery_manager).start.once
+    mock.proxy(d).timer_execute(:service_discovery_helper_test, anything).never
+
+    d.start
+    d.event_loop_wait_until_start
+
+    assert_equal 1, d.discovery_manager.services.size
+    d.select_service do |serv|
+      assert_equal "127.0.0.1", serv.host
+      assert_equal 1234, serv.port
+    end
+  end
+
+  test 'call timer_execute if dynamic configuration' do
+    d = @d = DummyPlugin.new
+    d.configure(config_element('root', '', {}, [config_element('service_discovery', '', { '@type' => 'file', 'path' => File.join(@sd_file_dir, 'config.yml' )})]))
 
     assert_true !!d.discovery_manager
     mock.proxy(d.discovery_manager).start.once
@@ -71,25 +106,22 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
   end
 
   test 'exits service discovery instances without any errors' do
-    d = @d = Dummy.new
+    d = @d = DummyPlugin.new
     mockv = flexmock('dns_resolver', getaddress: '127.0.0.1')
               .should_receive(:getresources)
               .and_return([Resolv::DNS::Resource::IN::SRV.new(1, 10, 8081, 'service1.example.com')])
               .mock
     mock(Resolv::DNS).new { mockv }
 
-    d.service_discovery_create_manager(
-      :service_discovery_helper_test2,
-      configurations: [{ type: :srv, conf: config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com' }) }],
-    )
+    d.configure(config_element('root', '', {}, [config_element('service_discovery', '', { '@type' => 'srv', 'service' => 'service1', 'hostname' => 'example.com' })]))
 
     assert_true !!d.discovery_manager
     mock.proxy(d.discovery_manager).start.once
-    mock(d).timer_execute(:service_discovery_helper_test2, anything).once
+    mock(d).timer_execute(:service_discovery_helper_test, anything).once
 
     # To avoid claring `@logs` during `terminate` step
     # https://github.com/fluent/fluentd/blob/bc78d889f93dad8c2a4e0ad1ca802546185dacba/lib/fluent/test/log.rb#L33
-    mock(d.log).reset.twice
+    mock(d.log).reset.times(3)
 
     d.start
     d.event_loop_wait_until_start
@@ -102,4 +134,32 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
 
     assert_false(d.log.out.logs.any? { |e| e.match?(/thread doesn't exit correctly/) })
   end
+
+  test 'static service discovery will be configured automatically when default service directive is specified' do
+    d = @d = DummyPlugin.new
+
+    nodes = [
+      config_element('node', '', { 'host' => '192.168.0.1', 'port' => '24224' }),
+      config_element('node', '', { 'host' => '192.168.0.2', 'port' => '24224' })
+    ]
+    d.configure(config_element('root', '', {}, nodes))
+
+    assert_true !!d.discovery_manager
+
+    mock.proxy(d.discovery_manager).start.once
+    mock.proxy(d).timer_execute(:service_discovery_helper_test, anything).never
+
+    d.start
+    d.event_loop_wait_until_start
+
+    assert_equal 2, d.discovery_manager.services.size
+    d.select_service do |serv|
+      assert_equal "192.168.0.1", serv.host
+      assert_equal 24224, serv.port
+    end
+    d.select_service do |serv|
+      assert_equal "192.168.0.2", serv.host
+      assert_equal 24224, serv.port
+    end
+  end
 end

data/test/plugin_helper/test_socket.rb

@@ -6,17 +6,24 @@ require 'socket'
 require 'openssl'
 
 class SocketHelperTest < Test::Unit::TestCase
-  PORT = unused_port
   CERT_DIR = File.expand_path(File.dirname(__FILE__) + '/data/cert/without_ca')
   CA_CERT_DIR = File.expand_path(File.dirname(__FILE__) + '/data/cert/with_ca')
   CERT_CHAINS_DIR = File.expand_path(File.dirname(__FILE__) + '/data/cert/cert_chains')
 
+  def setup
+    @port = unused_port
+  end
+
+  def teardown
+    @port = nil
+  end
+
   class SocketHelperTestPlugin < Fluent::Plugin::TestBase
     helpers :socket
   end
 
   class EchoTLSServer
-    def initialize(host = '127.0.0.1', port = PORT, cert_path: nil, private_key_path: nil, ca_path: nil)
+    def initialize(port, host: '127.0.0.1', cert_path: nil, private_key_path: nil, ca_path: nil)
       server = TCPServer.open(host, port)
       ctx = OpenSSL::SSL::SSLContext.new
       ctx.cert = OpenSSL::X509::Certificate.new(File.open(cert_path)) if cert_path
@@ -91,8 +98,8 @@ class SocketHelperTest < Test::Unit::TestCase
     cert_path = File.join(CERT_DIR, 'cert.pem')
     private_key_path = File.join(CERT_DIR, 'cert-key.pem')
 
-    EchoTLSServer.new(cert_path: cert_path, private_key_path: private_key_path).start do
-      client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', PORT, verify_fqdn: false, cert_paths: [cert_path])
+    EchoTLSServer.new(@port, cert_path: cert_path, private_key_path: private_key_path).start do
+      client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', @port, verify_fqdn: false, cert_paths: [cert_path])
       client.write('hello')
       assert_equal 'hello', client.readpartial(100)
       client.close
@@ -105,8 +112,8 @@ class SocketHelperTest < Test::Unit::TestCase
 
     ca_cert_path = File.join(CA_CERT_DIR, 'ca-cert.pem')
 
-    EchoTLSServer.new(cert_path: cert_path, private_key_path: private_key_path).start do
-      client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', PORT, verify_fqdn: false, cert_paths: [ca_cert_path])
+    EchoTLSServer.new(@port, cert_path: cert_path, private_key_path: private_key_path).start do
+      client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', @port, verify_fqdn: false, cert_paths: [ca_cert_path])
       client.write('hello')
       assert_equal 'hello', client.readpartial(100)
       client.close
@@ -121,8 +128,8 @@ class SocketHelperTest < Test::Unit::TestCase
     client_cert_path = File.join(CERT_CHAINS_DIR, 'cert.pem')
     client_private_key_path = File.join(CERT_CHAINS_DIR, 'cert-key.pem')
 
-    EchoTLSServer.new(cert_path: cert_path, private_key_path: private_key_path, ca_path: client_ca_cert_path).start do
-      client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', PORT, verify_fqdn: false, cert_path: client_cert_path, private_key_path: client_private_key_path, cert_paths: [cert_path])
+    EchoTLSServer.new(@port, cert_path: cert_path, private_key_path: private_key_path, ca_path: client_ca_cert_path).start do
+      client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', @port, verify_fqdn: false, cert_path: client_cert_path, private_key_path: client_private_key_path, cert_paths: [cert_path])
       client.write('hello')
       assert_equal 'hello', client.readpartial(100)
       client.close
@@ -133,7 +140,7 @@ class SocketHelperTest < Test::Unit::TestCase
     cert_path = File.expand_path(File.dirname(__FILE__) + '/data/cert/empty.pem')
 
     assert_raise Fluent::ConfigError do
-      SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', PORT, cert_path: cert_path)
+      SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', @port, cert_path: cert_path)
     end
   end
 end

data/test/test_config.rb

@@ -160,7 +160,8 @@ class ConfigTest < Test::Unit::TestCase
     prepare_config
     opts = {
       :config_path => "#{TMP_DIR}/config_test_1.conf",
-      :inline_config => "<source>\n  type http\n  port 2222\n </source>"
+      :inline_config => "<source>\n  type http\n  port 2222\n </source>",
+      :use_v1_config => false
     }
     assert_nothing_raised do
       Fluent::Supervisor.new(opts)

data/test/test_event.rb

@@ -401,6 +401,22 @@ module EventTest
         i += 1
       }
     end
+
+    # `any?` represents an Enumerable method which calls `each` internally
+    test 'size_after_any' do
+      @es.any?
+
+      assert_equal 2, @es.size
+    end
+
+    # `any?` represents an Enumerable method which calls `each` internally
+    test 'each_after_any' do
+      @es.any?
+
+      count = 0
+      @es.each { |time, record| count += 1 }
+      assert_equal 2, count
+    end
   end
 
   class CompressedMessagePackEventStreamTest < ::Test::Unit::TestCase

data/test/test_formatter.rb

@@ -184,6 +184,36 @@ module FormatterTest
 
       assert_equal("message=awesome,greeting=hello#{@newline}", formatted)
     end
+
+    def record_with_tab
+      {'message' => "awe\tsome", 'greeting' => "hello\t"}
+    end
+
+    def test_format_suppresses_tab
+      @formatter.configure({})
+      formatted = @formatter.format(tag, @time, record_with_tab)
+
+      assert_equal("message:awe some\tgreeting:hello #{@newline}", formatted)
+    end
+
+    def test_format_suppresses_tab_custom_replacement
+      @formatter.configure(
+        'replacement'      => 'X',
+      )
+      formatted = @formatter.format(tag, @time, record_with_tab)
+
+      assert_equal("message:aweXsome\tgreeting:helloX#{@newline}", formatted)
+    end
+
+    def test_format_suppresses_custom_delimiter
+      @formatter.configure(
+        'delimiter'       => 'w',
+        'label_delimiter' => '=',
+      )
+      formatted = @formatter.format(tag, @time, record)
+
+      assert_equal("message=a esomewgreeting=hello#{@newline}", formatted)
+    end
   end
 
   class CsvFormatterTest < ::Test::Unit::TestCase

data/test/test_output.rb

@@ -117,7 +117,7 @@ module FluentOutputTest
       end
 
       mock(d.instance.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
-                                { primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output" })
+                                primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output")
       d.configure(CONFIG + %[
         <secondary>
           type test2
@@ -133,7 +133,7 @@ module FluentOutputTest
       d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::ObjectBufferedOutput)
 
       mock(d.instance.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
-                                { primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output" }).never
+                                primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output").never
       d.configure(CONFIG + %[
         <secondary>
           type test2

data/test/test_supervisor.rb

@@ -7,6 +7,7 @@ require_relative 'test_plugin_classes'
 require 'net/http'
 require 'uri'
 require 'fileutils'
+require 'tempfile'
 
 if Fluent.windows?
   require 'win32/event'
@@ -489,6 +490,40 @@ class SupervisorTest < ::Test::Unit::TestCase
     assert_equal 10, $log.out.instance_variable_get(:@shift_size)
   end
 
+  sub_test_case "system log rotation" do
+    def parse_text(text)
+      basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
+      Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }
+    end
+
+    def test_override_default_log_rotate
+      Tempfile.open do |file|
+        config = parse_text(<<-EOS)
+          <system>
+            <log>
+              rotate_age 3
+              rotate_size 300
+            </log>
+          </system>
+        EOS
+        file.puts(config)
+        file.flush
+        opts = Fluent::Supervisor.default_options.merge(
+          log_path: "#{TMP_DIR}/test.log", config_path: file.path
+        )
+        sv = Fluent::Supervisor.new(opts)
+
+        log = sv.instance_variable_get(:@log)
+        log.init(:standalone, 0)
+        logger = $log.instance_variable_get(:@logger)
+
+        assert_equal([3, 300],
+                     [logger.instance_variable_get(:@rotate_age),
+                      logger.instance_variable_get(:@rotate_size)])
+      end
+    end
+  end
+
   def test_inline_config
     omit 'this feature is deprecated. see https://github.com/fluent/fluentd/issues/2711'
 

data/test/test_time_parser.rb

@@ -228,4 +228,113 @@ class TimeParserTest < ::Test::Unit::TestCase
       assert_equal_event_time(time, parser.parse("#{time.sec}.#{time.nsec}"))
     end
   end
+
+  sub_test_case 'MixedTimeParser fallback' do
+    class DummyForTimeParser
+      include Fluent::Configurable
+      include Fluent::TimeMixin::Parser
+    end
+
+    test 'no time_format_fallbacks failure' do
+      i = DummyForTimeParser.new
+      assert_raise(Fluent::ConfigError.new("time_type is :mixed but time_format and time_format_fallbacks is empty.")) do
+        i.configure(config_element('parse', '', {'time_type' => 'mixed'}))
+      end
+    end
+
+    test 'fallback time format failure' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format_fallbacks' => ['%iso8601']}))
+      parser = i.time_parser_create
+      assert_raise(Fluent::TimeParser::TimeParseError.new("invalid time format: value = INVALID, even though fallbacks: Fluent::TimeParser")) do
+        parser.parse("INVALID")
+      end
+    end
+
+    test 'primary format is unixtime, secondary %iso8601 is used' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => 'unixtime',
+                                  'time_format_fallbacks' => ['%iso8601']}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(time, parser.parse('2021-01-01T12:00:00+0900'))
+    end
+
+    test 'primary format is %iso8601, secondary unixtime is used' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => '%iso8601',
+                                  'time_format_fallbacks' => ['unixtime']}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(time, parser.parse("#{time.sec}"))
+    end
+
+    test 'primary format is %iso8601, no secondary is used' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => '%iso8601'}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(time, parser.parse("2021-01-01T12:00:00+0900"))
+    end
+
+    test 'primary format is unixtime, no secondary is used' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => 'unixtime'}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(time, parser.parse("#{time.sec}"))
+    end
+
+    test 'primary format is %iso8601, raise error because of no appropriate secondary' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => '%iso8601'}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_raise("Fluent::TimeParser::TimeParseError: invalid time format: value = #{time.sec}, even though fallbacks: Fluent::TimeParser") do
+        parser.parse("#{time.sec}")
+      end
+    end
+
+    test 'primary format is unixtime, raise error because of no appropriate secondary' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => 'unixtime'}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_raise("Fluent::TimeParser::TimeParseError: invalid time format: value = #{time}, even though fallbacks: Fluent::NumericTimeParser") do
+        parser.parse("2021-01-01T12:00:00+0900")
+      end
+    end
+
+    test 'fallback to unixtime' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '', {'time_type' => 'mixed',
+                                               'time_format_fallbacks' => ['%iso8601', 'unixtime']}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(Fluent::EventTime.new(time.to_i), parser.parse("#{time.sec}"))
+    end
+
+    test 'fallback to %iso8601' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '', {'time_type' => 'mixed',
+                                               'time_format_fallbacks' => ['unixtime', '%iso8601']}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(time, parser.parse('2021-01-01T12:00:00+0900'))
+    end
+  end
 end