fluentd 1.9.3 → 1.10.0

data/test/plugin/test_sd_file.rb

@@ -149,6 +149,23 @@ class FileServiceDiscoveryTest < ::Test::Unit::TestCase
       assert_empty queue
     end
 
+    test 'Skip if error is occured' do
+      @sd_file.extend(TestStatEventHelperWrapper)
+
+      create_tmp_config('config.json', JSON.generate([{ port: 1233, host: '127.0.0.1' }]))
+      @sd_file.configure(config_element('service_discovery', '', { 'path' => File.join(@dir, 'config.yml') }))
+      queue = []
+
+      FileUtils.rm_r(File.join(@dir, 'tmp', 'config.json'))
+      mock.proxy(@sd_file).refresh_file(queue).twice
+
+      @sd_file.start(queue)
+      assert_empty queue
+
+      @sd_file.resume
+      assert_empty queue
+    end
+
     test 'if service is updated, service_in and service_out event happen' do
       @sd_file.extend(TestStatEventHelperWrapper)
 

data/test/plugin/test_sd_srv.rb

@@ -0,0 +1,230 @@
+require_relative '../helper'
+require 'fluent/plugin/sd_srv'
+require 'fileutils'
+require 'flexmock/test_unit'
+require 'json'
+
+class SrvServiceDiscoveryTest < ::Test::Unit::TestCase
+  SRV_RECORD1 = Resolv::DNS::Resource::IN::SRV.new(1, 10, 8081, 'service1.example.com')
+  SRV_RECORD2 = Resolv::DNS::Resource::IN::SRV.new(2, 20, 8082, 'service2.example.com')
+
+  sub_test_case 'configure' do
+    test 'set services ordered by priority' do
+      sdf = Fluent::Plugin::SrvServiceDiscovery.new
+      mock(Resolv::DNS).new { flexmock('dns_resolver', getresources: [SRV_RECORD2, SRV_RECORD1], getaddress: '127.0.0.1') }
+
+      sdf.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com' }))
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8081, 'service1.example.com', 10, false, '', '', nil), sdf.services[0]
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8082, 'service2.example.com', 20, false, '', '', nil), sdf.services[1]
+    end
+
+    test 'reuturn host name without revolving name when dns_lookup is flase' do
+      sdf = Fluent::Plugin::SrvServiceDiscovery.new
+      mock(Resolv::DNS).new { flexmock('dns_resolver', getresources: [SRV_RECORD1, SRV_RECORD2], getaddress: '127.0.0.1') }
+
+      sdf.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'dns_lookup' => false }))
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, 'service1.example.com', 8081, 'service1.example.com', 10, false, '', '', nil), sdf.services[0]
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, 'service2.example.com', 8082, 'service2.example.com', 20, false, '', '', nil), sdf.services[1]
+    end
+
+    test 'pass a value as :nameserver to Resolve::DNS when dns_server_host is given' do
+      sdf = Fluent::Plugin::SrvServiceDiscovery.new
+      mock(Resolv::DNS).new(nameserver: '8.8.8.8') { flexmock('dns_resolver', getresources: [SRV_RECORD1, SRV_RECORD2], getaddress: '127.0.0.1') }
+
+      sdf.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'dns_server_host' => '8.8.8.8' }))
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8081, 'service1.example.com', 10, false, '', '', nil), sdf.services[0]
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8082, 'service2.example.com', 20, false, '', '', nil), sdf.services[1]
+    end
+
+    test 'pass a value as :nameserver_port to Resolve::DNS when dns_server_host has port' do
+      sdf = Fluent::Plugin::SrvServiceDiscovery.new
+      mock(Resolv::DNS).new(nameserver_port: [['8.8.8.8', 8080]]) { flexmock('dns_resolver', getresources: [SRV_RECORD1, SRV_RECORD2], getaddress: '127.0.0.1') }
+
+      sdf.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'dns_server_host' => '8.8.8.8:8080' }))
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8081, 'service1.example.com', 10, false, '', '', nil), sdf.services[0]
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8082, 'service2.example.com', 20, false, '', '', nil), sdf.services[1]
+    end
+
+    test 'target follows RFC2782' do
+      sdf = Fluent::Plugin::SrvServiceDiscovery.new
+      mock = flexmock('dns_resolver', getaddress: '127.0.0.1')
+               .should_receive(:getresources).with("_service1._tcp.example.com", Resolv::DNS::Resource::IN::SRV)
+               .and_return([SRV_RECORD1, SRV_RECORD2])
+               .mock
+
+      mock(Resolv::DNS).new { mock }
+      sdf.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com' }))
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8081, 'service1.example.com', 10, false, '', '', nil), sdf.services[0]
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8082, 'service2.example.com', 20, false, '', '', nil), sdf.services[1]
+    end
+
+    test 'can change protocol' do
+      sdf = Fluent::Plugin::SrvServiceDiscovery.new
+      mock = flexmock('dns_resolver', getaddress: '127.0.0.1')
+               .should_receive(:getresources).with("_service1._udp.example.com", Resolv::DNS::Resource::IN::SRV)
+               .and_return([SRV_RECORD1, SRV_RECORD2])
+               .mock
+
+      mock(Resolv::DNS).new { mock }
+      sdf.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'proto' => 'udp' }))
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8081, 'service1.example.com', 10, false, '', '', nil), sdf.services[0]
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8082, 'service2.example.com', 20, false, '', '', nil), sdf.services[1]
+    end
+
+    test 'can set password, username, password' do
+      sdf = Fluent::Plugin::SrvServiceDiscovery.new
+      mock(Resolv::DNS).new { flexmock('dns_resolver', getresources: [SRV_RECORD2, SRV_RECORD1], getaddress: '127.0.0.1') }
+
+      sdf.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'shared_key' => 'key', 'username' => 'user', 'password' => 'pass' }))
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8081, 'service1.example.com', 10, false, 'user', 'pass', 'key'), sdf.services[0]
+      assert_equal Fluent::Plugin::ServiceDiscovery::Service.new(:srv, '127.0.0.1', 8082, 'service2.example.com', 20, false, 'user', 'pass', 'key'), sdf.services[1]
+    end
+  end
+
+  sub_test_case '#start' do
+    module TestTimerEventHelperWrapper
+      # easy to control statsevent
+      def timer_execute(_name, _interval, &block)
+        @test_timer_event_helper_wrapper_context = Fiber.new do
+          loop do
+            block.call
+
+            if Fiber.yield == :finish
+              break
+            end
+          end
+        end
+
+        resume
+      end
+
+      def resume
+        @test_timer_event_helper_wrapper_context.resume(:resume)
+      end
+
+      def shutdown
+        super
+
+        if @test_timer_event_helper_wrapper_context
+          @test_timer_event_helper_wrapper_context.resume(:finish)
+        end
+
+      end
+    end
+
+    setup do
+      sds = Fluent::Plugin::SrvServiceDiscovery.new
+      @sd_srv = sds
+    end
+
+    teardown do
+      if @sd_srv
+        @sd_srv.stop unless @sd_srv.stopped?
+        @sd_srv.before_shutdown unless @sd_srv.before_shutdown?
+        @sd_srv.shutdown unless @sd_srv.shutdown?
+        @sd_srv.after_shutdown unless @sd_srv.after_shutdown?
+        @sd_srv.close unless @sd_srv.closed?
+        @sd_srv.terminate unless @sd_srv.terminated?
+      end
+    end
+
+    test 'Skip if srv record is not updated' do
+      @sd_srv.extend(TestTimerEventHelperWrapper)
+      mock(Resolv::DNS).new { flexmock('dns_resolver', getresources: [SRV_RECORD2, SRV_RECORD1], getaddress: '127.0.0.1') }
+      @sd_srv.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com' }))
+      queue = []
+
+      @sd_srv.start(queue)
+      assert_empty queue
+
+      @sd_srv.resume
+      assert_empty queue
+    end
+
+    test 'Skip if DNS resolver raise an error' do
+      @sd_srv.extend(TestTimerEventHelperWrapper)
+      mock = flexmock('dns_resolver', getaddress: '127.0.0.1')
+               .should_receive(:getresources)
+               .and_return([SRV_RECORD1, SRV_RECORD2])
+               .and_return { raise 'some error' } # for start
+               .and_return { raise 'some error' } # for resume
+               .mock
+
+      mock(Resolv::DNS).new { mock }
+      @sd_srv.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com' }))
+      queue = []
+
+      @sd_srv.start(queue)
+      assert_empty queue
+
+      @sd_srv.resume
+      assert_empty queue
+    end
+
+    test 'if service is updated, service_in and service_out event happen' do
+      @sd_srv.extend(TestTimerEventHelperWrapper)
+      mock = flexmock('dns_resolver', getaddress: '127.0.0.1')
+               .should_receive(:getresources)
+               .and_return([SRV_RECORD1])
+               .and_return([SRV_RECORD2])
+               .mock
+
+      mock(Resolv::DNS).new { mock }
+      @sd_srv.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'dns_lookup' => false }))
+      queue = []
+
+      @sd_srv.start(queue)
+      join = queue.shift
+      drain = queue.shift
+      assert_equal Fluent::Plugin::ServiceDiscovery::SERVICE_IN, join.type
+      assert_equal 8082, join.service.port
+      assert_equal 'service2.example.com', join.service.host
+
+      assert_equal Fluent::Plugin::ServiceDiscovery::SERVICE_OUT, drain.type
+      assert_equal 8081, drain.service.port
+      assert_equal 'service1.example.com', drain.service.host
+    end
+
+    test 'if service is deleted, service_out event happens' do
+      @sd_srv.extend(TestTimerEventHelperWrapper)
+      mock = flexmock('dns_resolver', getaddress: '127.0.0.1')
+               .should_receive(:getresources)
+               .and_return([SRV_RECORD1, SRV_RECORD2])
+               .and_return([SRV_RECORD2])
+               .mock
+
+      mock(Resolv::DNS).new { mock }
+      @sd_srv.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'dns_lookup' => false }))
+      queue = []
+
+      @sd_srv.start(queue)
+
+      assert_equal 1, queue.size
+      drain = queue.shift
+      assert_equal Fluent::Plugin::ServiceDiscovery::SERVICE_OUT, drain.type
+      assert_equal 8081, drain.service.port
+      assert_equal 'service1.example.com', drain.service.host
+    end
+
+    test 'if new service is added, service_in event happens' do
+      @sd_srv.extend(TestTimerEventHelperWrapper)
+      mock = flexmock('dns_resolver', getaddress: '127.0.0.1')
+               .should_receive(:getresources)
+               .and_return([SRV_RECORD2])
+               .and_return([SRV_RECORD1, SRV_RECORD2])
+               .mock
+
+      mock(Resolv::DNS).new { mock }
+      @sd_srv.configure(config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com', 'dns_lookup' => false }))
+      queue = []
+
+      @sd_srv.start(queue)
+
+      assert_equal 1, queue.size
+      join = queue.shift
+      assert_equal Fluent::Plugin::ServiceDiscovery::SERVICE_IN, join.type
+      assert_equal 8081, join.service.port
+      assert_equal 'service1.example.com', join.service.host
+    end
+  end
+end

data/test/plugin_helper/data/cert/cert-with-CRLF.pem

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDLDCCAhSgAwIBAgIIEJHFsHrKBGYwDQYJKoZIhvcNAQELBQAwJjEQMA4GA1UE
+ChMHRmx1ZW50ZDESMBAGA1UEAxMJbG9jYWxob3N0MB4XDTE5MDYxOTA1MTM0NVoX
+DTE5MDkxODExMTg0NVowJjEQMA4GA1UEChMHRmx1ZW50ZDESMBAGA1UEAxMJbG9j
+YWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArbUTk5n5RruI
+QGhK1T8g/emKonlfWNMSj/J/f/U9NJ14ugIxyHBMqx4WaTyA4zjT2VJO5tRBe385
+zlIUf8i+x7Ovt/MgsjiwXyKv7qdsE5KHLq+VXJfA+s5vAAyzBHY/BA7xxh/QqCI8
+a/a1OyHyaQ9pFRFXtQBlTH7Fc1qSw5Yg0EXofa6YIBQuDjfqa7FRPj+bEWDO5PUq
+OMzH5XKBUPS9GLHOqia0CnzF2a51TArC0Dl1oNFa7myVmjBuNtkG88Fkd7YNzGa+
+sNBJPmuGvFXuU3XPEnrtARO/SG4g9/MQUvfMI3jFFOJAMmEKd8QXxO5FgIuEnaNO
+hVRVH/e9wwIDAQABo14wXDAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYB
+BQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFPLdOVnVWuuB
+7Pnvpgte4BHitzFYMA0GCSqGSIb3DQEBCwUAA4IBAQBp8LAzjWIJapwTBnnivwZk
+D6Lr028mZIacbBZKsmmPi0VvDFqCvUAbHN8ytPlRBWnvvkihDkZs1TwcDCXGsWYs
+dNNwsYGpk3mQxsHQ9atvy0mQGLDlfaSs/329bfVCw1cPFo9n+MeivSBoE6asdIbH
+tOW3kk1XtJZ2qQJJRvexFImZc0z8c2cG0+eR5hQxQd9bLnAczi/8mZ8VzaU/O3UU
+OJoVuyp0AA8f2f0f1QDaeH9stWZtJQj3ZX1DWHRE3OmVkoBdlt8EHYGggtvQaLIF
+XbHigLHzYztMjmDt4fmRczu/Fu6M4xNro8jLgjiIjqlLBjDZiKrSbOwgyebwFDlv
+-----END CERTIFICATE-----

data/test/plugin_helper/test_cert_option.rb

@@ -12,5 +12,7 @@ class CertOptionPluginHelperTest < Test::Unit::TestCase
     assert_equal(1, certs.length)
     certs = d.cert_option_certificates_from_file("test/plugin_helper/data/cert/cert-with-no-newline.pem")
     assert_equal(1, certs.length)
+    certs = d.cert_option_certificates_from_file("test/plugin_helper/data/cert/cert-with-CRLF.pem")
+    assert_equal(1, certs.length)
   end
 end

data/test/plugin_helper/test_child_process.rb

@@ -286,9 +286,11 @@ class ChildProcessTest < Test::Unit::TestCase
       assert_equal [], @d.log.out.logs
 
       @d.stop # nothing occurs
-      sleep TEST_WAIT_INTERVAL_FOR_LOOP
-      lines1 = ary.size
-      assert{ lines1 > 1 }
+      lines1 = nil
+      waiting(TEST_WAIT_INTERVAL_FOR_LOOP * 3) do
+        lines1 = ary.size
+        lines1 > 1
+      end
 
       pid = @d._child_process_processes.keys.first
       # default value 10 is too long for test

data/test/test_msgpack_factory.rb

@@ -0,0 +1,18 @@
+require_relative 'helper'
+require 'fluent/msgpack_factory'
+
+class MessagePackFactoryTest < Test::Unit::TestCase
+  test 'call log.warn only once' do
+    klass = Class.new do
+      include Fluent::MessagePackFactory::Mixin
+    end
+
+    mp = klass.new
+
+    mock.proxy($log).warn(anything).once
+
+    assert mp.msgpack_factory
+    assert mp.msgpack_factory
+    assert mp.msgpack_factory
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.9.3
+  version: 1.10.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-05 00:00:00.000000000 Z
+date: 2020-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -359,6 +359,7 @@ files:
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/feature_request.md"
 - ".github/PULL_REQUEST_TEMPLATE.md"
+- ".github/workflows/issue-auto-closer.yml"
 - ".gitignore"
 - ".gitlab-ci.yml"
 - ".travis.yml"
@@ -576,6 +577,7 @@ files:
 - lib/fluent/plugin/parser_syslog.rb
 - lib/fluent/plugin/parser_tsv.rb
 - lib/fluent/plugin/sd_file.rb
+- lib/fluent/plugin/sd_srv.rb
 - lib/fluent/plugin/sd_static.rb
 - lib/fluent/plugin/service_discovery.rb
 - lib/fluent/plugin/socket_util.rb
@@ -707,6 +709,8 @@ files:
 - test/plugin/data/sd_file/config.yaml
 - test/plugin/data/sd_file/config.yml
 - test/plugin/data/sd_file/invalid_config.yml
+- test/plugin/in_tail/test_fifo.rb
+- test/plugin/in_tail/test_io_handler.rb
 - test/plugin/in_tail/test_position_file.rb
 - test/plugin/out_forward/test_ack_handler.rb
 - test/plugin/out_forward/test_connection_manager.rb
@@ -789,10 +793,12 @@ files:
 - test/plugin/test_parser_syslog.rb
 - test/plugin/test_parser_tsv.rb
 - test/plugin/test_sd_file.rb
+- test/plugin/test_sd_srv.rb
 - test/plugin/test_storage.rb
 - test/plugin/test_storage_local.rb
 - test/plugin/test_string_util.rb
 - test/plugin_helper/data/cert/cert-key.pem
+- test/plugin_helper/data/cert/cert-with-CRLF.pem
 - test/plugin_helper/data/cert/cert-with-no-newline.pem
 - test/plugin_helper/data/cert/cert.pem
 - test/plugin_helper/data/cert/generate_cert.rb
@@ -851,6 +857,7 @@ files:
 - test/test_logger_initializer.rb
 - test/test_match.rb
 - test/test_mixin.rb
+- test/test_msgpack_factory.rb
 - test/test_output.rb
 - test/test_plugin.rb
 - test/test_plugin_classes.rb
@@ -929,6 +936,8 @@ test_files:
 - test/plugin/data/sd_file/config.yaml
 - test/plugin/data/sd_file/config.yml
 - test/plugin/data/sd_file/invalid_config.yml
+- test/plugin/in_tail/test_fifo.rb
+- test/plugin/in_tail/test_io_handler.rb
 - test/plugin/in_tail/test_position_file.rb
 - test/plugin/out_forward/test_ack_handler.rb
 - test/plugin/out_forward/test_connection_manager.rb
@@ -1011,10 +1020,12 @@ test_files:
 - test/plugin/test_parser_syslog.rb
 - test/plugin/test_parser_tsv.rb
 - test/plugin/test_sd_file.rb
+- test/plugin/test_sd_srv.rb
 - test/plugin/test_storage.rb
 - test/plugin/test_storage_local.rb
 - test/plugin/test_string_util.rb
 - test/plugin_helper/data/cert/cert-key.pem
+- test/plugin_helper/data/cert/cert-with-CRLF.pem
 - test/plugin_helper/data/cert/cert-with-no-newline.pem
 - test/plugin_helper/data/cert/cert.pem
 - test/plugin_helper/data/cert/generate_cert.rb
@@ -1073,6 +1084,7 @@ test_files:
 - test/test_logger_initializer.rb
 - test/test_match.rb
 - test/test_mixin.rb
+- test/test_msgpack_factory.rb
 - test/test_output.rb
 - test/test_plugin.rb
 - test/test_plugin_classes.rb