fluentd 1.12.2 → 1.13.2

data/test/plugin_helper/test_service_discovery.rb

@@ -17,6 +17,24 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
     end
   end
 
+  class DummyPlugin < Fluent::Plugin::TestBase
+    helpers :service_discovery
+
+    def configure(conf)
+      super
+      service_discovery_configure(:service_discovery_helper_test, static_default_service_directive: 'node')
+    end
+
+    def select_service(&block)
+      service_discovery_select_service(&block)
+    end
+
+    # Make these mehtod public
+    def discovery_manager
+      super
+    end
+  end
+
   setup do
     @sd_file_dir = File.expand_path('../plugin/data/sd_file', __dir__)
 
@@ -33,7 +51,7 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
     end
   end
 
-  test 'start discovery manager' do
+  test 'support calling #service_discovery_create_manager and #discovery_manager from plugin' do
     d = @d = Dummy.new
 
     d.service_discovery_create_manager(
@@ -55,13 +73,30 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
     assert_equal 1234, services[0].port
   end
 
-  test 'call timer_execute if dynamic configuration' do
-    d = @d = Dummy.new
+  test 'start discovery manager' do
+    d = @d = DummyPlugin.new
 
-    d.service_discovery_create_manager(
-      :service_discovery_helper_test,
-      configurations: [{ type: :file, conf: config_element('file_config', '', { 'path' => File.join(@sd_file_dir, 'config.yml') }) }],
-    )
+    services = [config_element('service', '', { 'host' => '127.0.0.1', 'port' => '1234' })]
+    d.configure(config_element('root', '', {}, [config_element('service_discovery', '', {'@type' => 'static'}, services)]))
+
+    assert_true !!d.discovery_manager
+
+    mock.proxy(d.discovery_manager).start.once
+    mock.proxy(d).timer_execute(:service_discovery_helper_test, anything).never
+
+    d.start
+    d.event_loop_wait_until_start
+
+    assert_equal 1, d.discovery_manager.services.size
+    d.select_service do |serv|
+      assert_equal "127.0.0.1", serv.host
+      assert_equal 1234, serv.port
+    end
+  end
+
+  test 'call timer_execute if dynamic configuration' do
+    d = @d = DummyPlugin.new
+    d.configure(config_element('root', '', {}, [config_element('service_discovery', '', { '@type' => 'file', 'path' => File.join(@sd_file_dir, 'config.yml' )})]))
 
     assert_true !!d.discovery_manager
     mock.proxy(d.discovery_manager).start.once
@@ -71,25 +106,22 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
   end
 
   test 'exits service discovery instances without any errors' do
-    d = @d = Dummy.new
+    d = @d = DummyPlugin.new
     mockv = flexmock('dns_resolver', getaddress: '127.0.0.1')
               .should_receive(:getresources)
               .and_return([Resolv::DNS::Resource::IN::SRV.new(1, 10, 8081, 'service1.example.com')])
               .mock
     mock(Resolv::DNS).new { mockv }
 
-    d.service_discovery_create_manager(
-      :service_discovery_helper_test2,
-      configurations: [{ type: :srv, conf: config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com' }) }],
-    )
+    d.configure(config_element('root', '', {}, [config_element('service_discovery', '', { '@type' => 'srv', 'service' => 'service1', 'hostname' => 'example.com' })]))
 
     assert_true !!d.discovery_manager
     mock.proxy(d.discovery_manager).start.once
-    mock(d).timer_execute(:service_discovery_helper_test2, anything).once
+    mock(d).timer_execute(:service_discovery_helper_test, anything).once
 
     # To avoid claring `@logs` during `terminate` step
     # https://github.com/fluent/fluentd/blob/bc78d889f93dad8c2a4e0ad1ca802546185dacba/lib/fluent/test/log.rb#L33
-    mock(d.log).reset.twice
+    mock(d.log).reset.times(3)
 
     d.start
     d.event_loop_wait_until_start
@@ -102,4 +134,32 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
 
     assert_false(d.log.out.logs.any? { |e| e.match?(/thread doesn't exit correctly/) })
   end
+
+  test 'static service discovery will be configured automatically when default service directive is specified' do
+    d = @d = DummyPlugin.new
+
+    nodes = [
+      config_element('node', '', { 'host' => '192.168.0.1', 'port' => '24224' }),
+      config_element('node', '', { 'host' => '192.168.0.2', 'port' => '24224' })
+    ]
+    d.configure(config_element('root', '', {}, nodes))
+
+    assert_true !!d.discovery_manager
+
+    mock.proxy(d.discovery_manager).start.once
+    mock.proxy(d).timer_execute(:service_discovery_helper_test, anything).never
+
+    d.start
+    d.event_loop_wait_until_start
+
+    assert_equal 2, d.discovery_manager.services.size
+    d.select_service do |serv|
+      assert_equal "192.168.0.1", serv.host
+      assert_equal 24224, serv.port
+    end
+    d.select_service do |serv|
+      assert_equal "192.168.0.2", serv.host
+      assert_equal 24224, serv.port
+    end
+  end
 end

data/test/plugin_helper/test_socket.rb

@@ -6,17 +6,24 @@ require 'socket'
 require 'openssl'
 
 class SocketHelperTest < Test::Unit::TestCase
-  PORT = unused_port
   CERT_DIR = File.expand_path(File.dirname(__FILE__) + '/data/cert/without_ca')
   CA_CERT_DIR = File.expand_path(File.dirname(__FILE__) + '/data/cert/with_ca')
   CERT_CHAINS_DIR = File.expand_path(File.dirname(__FILE__) + '/data/cert/cert_chains')
 
+  def setup
+    @port = unused_port
+  end
+
+  def teardown
+    @port = nil
+  end
+
   class SocketHelperTestPlugin < Fluent::Plugin::TestBase
     helpers :socket
   end
 
   class EchoTLSServer
-    def initialize(host = '127.0.0.1', port = PORT, cert_path: nil, private_key_path: nil, ca_path: nil)
+    def initialize(port, host: '127.0.0.1', cert_path: nil, private_key_path: nil, ca_path: nil)
       server = TCPServer.open(host, port)
       ctx = OpenSSL::SSL::SSLContext.new
       ctx.cert = OpenSSL::X509::Certificate.new(File.open(cert_path)) if cert_path
@@ -91,8 +98,8 @@ class SocketHelperTest < Test::Unit::TestCase
     cert_path = File.join(CERT_DIR, 'cert.pem')
     private_key_path = File.join(CERT_DIR, 'cert-key.pem')
 
-    EchoTLSServer.new(cert_path: cert_path, private_key_path: private_key_path).start do
-      client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', PORT, verify_fqdn: false, cert_paths: [cert_path])
+    EchoTLSServer.new(@port, cert_path: cert_path, private_key_path: private_key_path).start do
+      client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', @port, verify_fqdn: false, cert_paths: [cert_path])
       client.write('hello')
       assert_equal 'hello', client.readpartial(100)
       client.close
@@ -105,8 +112,8 @@ class SocketHelperTest < Test::Unit::TestCase
 
     ca_cert_path = File.join(CA_CERT_DIR, 'ca-cert.pem')
 
-    EchoTLSServer.new(cert_path: cert_path, private_key_path: private_key_path).start do
-      client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', PORT, verify_fqdn: false, cert_paths: [ca_cert_path])
+    EchoTLSServer.new(@port, cert_path: cert_path, private_key_path: private_key_path).start do
+      client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', @port, verify_fqdn: false, cert_paths: [ca_cert_path])
       client.write('hello')
       assert_equal 'hello', client.readpartial(100)
       client.close
@@ -121,8 +128,8 @@ class SocketHelperTest < Test::Unit::TestCase
     client_cert_path = File.join(CERT_CHAINS_DIR, 'cert.pem')
     client_private_key_path = File.join(CERT_CHAINS_DIR, 'cert-key.pem')
 
-    EchoTLSServer.new(cert_path: cert_path, private_key_path: private_key_path, ca_path: client_ca_cert_path).start do
-      client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', PORT, verify_fqdn: false, cert_path: client_cert_path, private_key_path: client_private_key_path, cert_paths: [cert_path])
+    EchoTLSServer.new(@port, cert_path: cert_path, private_key_path: private_key_path, ca_path: client_ca_cert_path).start do
+      client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', @port, verify_fqdn: false, cert_path: client_cert_path, private_key_path: client_private_key_path, cert_paths: [cert_path])
       client.write('hello')
       assert_equal 'hello', client.readpartial(100)
       client.close
@@ -133,7 +140,7 @@ class SocketHelperTest < Test::Unit::TestCase
     cert_path = File.expand_path(File.dirname(__FILE__) + '/data/cert/empty.pem')
 
     assert_raise Fluent::ConfigError do
-      SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', PORT, cert_path: cert_path)
+      SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', @port, cert_path: cert_path)
     end
   end
 end

data/test/test_config.rb

@@ -160,7 +160,8 @@ class ConfigTest < Test::Unit::TestCase
     prepare_config
     opts = {
       :config_path => "#{TMP_DIR}/config_test_1.conf",
-      :inline_config => "<source>\n  type http\n  port 2222\n </source>"
+      :inline_config => "<source>\n  type http\n  port 2222\n </source>",
+      :use_v1_config => false
     }
     assert_nothing_raised do
       Fluent::Supervisor.new(opts)

data/test/test_event_time.rb

@@ -64,8 +64,8 @@ class EventTimeTest < Test::Unit::TestCase
 
   test 'Oj.dump' do
     time = Fluent::EventTime.new(100)
-    require 'fluent/env'
-    Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
+    require 'fluent/oj_options'
+    Fluent::OjOptions.load_env
     assert_equal('{"time":100}', Oj.dump({'time' => time}))
     assert_equal('["tag",100,{"key":"value"}]', Oj.dump(["tag", time, {"key" => "value"}], mode: :compat))
   end

data/test/test_oj_options.rb

@@ -0,0 +1,55 @@
+require_relative 'helper'
+require 'fluent/test'
+require 'fluent/oj_options'
+
+class OjOptionsTest < ::Test::Unit::TestCase
+  begin
+    require 'oj'
+    @@oj_is_avaibale = true
+  rescue LoadError
+    @@oj_is_avaibale = false
+  end
+
+  setup do
+    @orig_env = {}
+    ENV.each do |key, value|
+      @orig_env[key] = value if key.start_with?("FLUENT_OJ_OPTION_")
+    end
+  end
+
+  teardown do
+    ENV.delete_if { |key| key.start_with?("FLUENT_OJ_OPTION_") }
+    @orig_env.each { |key, value| ENV[key] = value }
+  end
+
+  test "available?" do
+    assert_equal(@@oj_is_avaibale, Fluent::OjOptions.available?)
+  end
+
+  sub_test_case "set by environment variable" do
+    test "when no env vars set, returns default options" do
+      ENV.delete_if { |key| key.start_with?("FLUENT_OJ_OPTION_") }
+      defaults = Fluent::OjOptions::DEFAULTS
+      assert_equal(defaults, Fluent::OjOptions.load_env)
+      assert_equal(defaults, Oj.default_options.slice(*defaults.keys)) if @@oj_is_avaibale
+    end
+
+    test "valid env var passed with valid value, default is overridden" do
+      ENV["FLUENT_OJ_OPTION_BIGDECIMAL_LOAD"] = ":bigdecimal"
+      assert_equal(:bigdecimal, Fluent::OjOptions.load_env[:bigdecimal_load])
+      assert_equal(:bigdecimal, Oj.default_options[:bigdecimal_load]) if @@oj_is_avaibale
+    end
+
+    test "valid env var passed with invalid value, default is not overriden" do
+      ENV["FLUENT_OJ_OPTION_BIGDECIMAL_LOAD"] = ":conor"
+      assert_equal(:float, Fluent::OjOptions.load_env[:bigdecimal_load])
+      assert_equal(:float, Oj.default_options[:bigdecimal_load]) if @@oj_is_avaibale
+    end
+
+    test "invalid env var passed, nothing done with it" do
+      ENV["FLUENT_OJ_OPTION_CONOR"] = ":conor"
+      assert_equal(nil, Fluent::OjOptions.load_env[:conor])
+      assert_equal(nil, Oj.default_options[:conor]) if @@oj_is_avaibale
+    end
+  end
+end

data/test/test_output.rb

@@ -117,7 +117,7 @@ module FluentOutputTest
       end
 
       mock(d.instance.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
-                                { primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output" })
+                                primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output")
       d.configure(CONFIG + %[
         <secondary>
           type test2
@@ -133,7 +133,7 @@ module FluentOutputTest
       d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::ObjectBufferedOutput)
 
       mock(d.instance.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
-                                { primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output" }).never
+                                primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output").never
       d.configure(CONFIG + %[
         <secondary>
           type test2

data/test/test_supervisor.rb

@@ -7,6 +7,7 @@ require_relative 'test_plugin_classes'
 require 'net/http'
 require 'uri'
 require 'fileutils'
+require 'tempfile'
 
 if Fluent.windows?
   require 'win32/event'
@@ -489,6 +490,40 @@ class SupervisorTest < ::Test::Unit::TestCase
     assert_equal 10, $log.out.instance_variable_get(:@shift_size)
   end
 
+  sub_test_case "system log rotation" do
+    def parse_text(text)
+      basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
+      Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }
+    end
+
+    def test_override_default_log_rotate
+      Tempfile.open do |file|
+        config = parse_text(<<-EOS)
+          <system>
+            <log>
+              rotate_age 3
+              rotate_size 300
+            </log>
+          </system>
+        EOS
+        file.puts(config)
+        file.flush
+        opts = Fluent::Supervisor.default_options.merge(
+          log_path: "#{TMP_DIR}/test.log", config_path: file.path
+        )
+        sv = Fluent::Supervisor.new(opts)
+
+        log = sv.instance_variable_get(:@log)
+        log.init(:standalone, 0)
+        logger = $log.instance_variable_get(:@logger)
+
+        assert_equal([3, 300],
+                     [logger.instance_variable_get(:@rotate_age),
+                      logger.instance_variable_get(:@rotate_size)])
+      end
+    end
+  end
+
   def test_inline_config
     omit 'this feature is deprecated. see https://github.com/fluent/fluentd/issues/2711'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.12.2
+  version: 1.13.2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-29 00:00:00.000000000 Z
+date: 2021-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -107,7 +107,7 @@ dependencies:
         version: 0.5.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.7.0
+        version: 0.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -117,7 +117,7 @@ dependencies:
         version: 0.5.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.7.0
+        version: 0.8.0
 - !ruby/object:Gem::Dependency
   name: sigdump
   requirement: !ruby/object:Gem::Requirement
@@ -268,14 +268,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -373,9 +373,9 @@ files:
 - ".deepsource.toml"
 - ".drone.yml"
 - ".github/ISSUE_TEMPLATE.md"
-- ".github/ISSUE_TEMPLATE/bug_report.md"
+- ".github/ISSUE_TEMPLATE/bug_report.yaml"
 - ".github/ISSUE_TEMPLATE/config.yml"
-- ".github/ISSUE_TEMPLATE/feature_request.md"
+- ".github/ISSUE_TEMPLATE/feature_request.yaml"
 - ".github/PULL_REQUEST_TEMPLATE.md"
 - ".github/workflows/issue-auto-closer.yml"
 - ".github/workflows/linux-test.yaml"
@@ -519,6 +519,7 @@ files:
 - lib/fluent/match.rb
 - lib/fluent/mixin.rb
 - lib/fluent/msgpack_factory.rb
+- lib/fluent/oj_options.rb
 - lib/fluent/output.rb
 - lib/fluent/output_chain.rb
 - lib/fluent/parser.rb
@@ -664,6 +665,7 @@ files:
 - lib/fluent/test/driver/multi_output.rb
 - lib/fluent/test/driver/output.rb
 - lib/fluent/test/driver/parser.rb
+- lib/fluent/test/driver/storage.rb
 - lib/fluent/test/driver/test_event_router.rb
 - lib/fluent/test/filter_test.rb
 - lib/fluent/test/formatter_test.rb
@@ -689,12 +691,14 @@ files:
 - templates/new_gem/lib/fluent/plugin/input.rb.erb
 - templates/new_gem/lib/fluent/plugin/output.rb.erb
 - templates/new_gem/lib/fluent/plugin/parser.rb.erb
+- templates/new_gem/lib/fluent/plugin/storage.rb.erb
 - templates/new_gem/test/helper.rb.erb
 - templates/new_gem/test/plugin/test_filter.rb.erb
 - templates/new_gem/test/plugin/test_formatter.rb.erb
 - templates/new_gem/test/plugin/test_input.rb.erb
 - templates/new_gem/test/plugin/test_output.rb.erb
 - templates/new_gem/test/plugin/test_parser.rb.erb
+- templates/new_gem/test/plugin/test_storage.rb.erb
 - templates/plugin_config_formatter/param.md-compact.erb
 - templates/plugin_config_formatter/param.md-table.erb
 - templates/plugin_config_formatter/param.md.erb
@@ -702,6 +706,7 @@ files:
 - test/command/test_binlog_reader.rb
 - test/command/test_ca_generate.rb
 - test/command/test_cap_ctl.rb
+- test/command/test_cat.rb
 - test/command/test_ctl.rb
 - test/command/test_fluentd.rb
 - test/command/test_plugin_config_formatter.rb
@@ -760,6 +765,7 @@ files:
 - test/plugin/test_buffer_memory_chunk.rb
 - test/plugin/test_compressable.rb
 - test/plugin/test_file_util.rb
+- test/plugin/test_file_wrapper.rb
 - test/plugin/test_filter.rb
 - test/plugin/test_filter_grep.rb
 - test/plugin/test_filter_parser.rb
@@ -897,6 +903,7 @@ files:
 - test/test_match.rb
 - test/test_mixin.rb
 - test/test_msgpack_factory.rb
+- test/test_oj_options.rb
 - test/test_output.rb
 - test/test_plugin.rb
 - test/test_plugin_classes.rb
@@ -931,7 +938,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Fluentd event collector
@@ -939,6 +946,7 @@ test_files:
 - test/command/test_binlog_reader.rb
 - test/command/test_ca_generate.rb
 - test/command/test_cap_ctl.rb
+- test/command/test_cat.rb
 - test/command/test_ctl.rb
 - test/command/test_fluentd.rb
 - test/command/test_plugin_config_formatter.rb
@@ -997,6 +1005,7 @@ test_files:
 - test/plugin/test_buffer_memory_chunk.rb
 - test/plugin/test_compressable.rb
 - test/plugin/test_file_util.rb
+- test/plugin/test_file_wrapper.rb
 - test/plugin/test_filter.rb
 - test/plugin/test_filter_grep.rb
 - test/plugin/test_filter_parser.rb
@@ -1134,6 +1143,7 @@ test_files:
 - test/test_match.rb
 - test/test_mixin.rb
 - test/test_msgpack_factory.rb
+- test/test_oj_options.rb
 - test/test_output.rb
 - test/test_plugin.rb
 - test/test_plugin_classes.rb