fluentd 1.10.2-x86-mingw32 → 1.10.3-x86-mingw32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2987fa7f12e342b3a20a9f374eac70214a7a5d4d1ba3cd471b1d0bec7eb8465d
-  data.tar.gz: b65963401cb734ce1a3bae1178d6add5277b563b0037a5afb4200acd3562cc50
+  metadata.gz: 4e50b4e23679ea753e495fb94888322669b6b0e58dd903d86f72ef65115300c3
+  data.tar.gz: 03de1387897feaf543fc41600b9b3a59beb59caff9cc5576b5d714a295198523
 SHA512:
-  metadata.gz: 1d5e3cef4e44db09112c559b099c095420af2b0e2de36c4d934da53ef6027f163a397e2a3ff19b70d8b3c10796377cd917d6a96f2ce701335725672c57b63da1
-  data.tar.gz: '0194915dc5d86d4a2d631e13fa127337fb8125a54f3689ca18851891fb0ae5f4e5e3598784c71d8ba7b5804a9e690ba977ce0dabcd9874b30633a73bd877771d'
+  metadata.gz: 131672fdf195a8c556a7476ce4fd6622adef9179595404138a206b7a0ce0dd9907d23c3926b56a92deac9522476f2de09d7ab38db5de8284ec4b6ae33a639092
+  data.tar.gz: 462342bad8e0a4800efe3fd25888415ac3e612924c902b2f622572972356403dac4cee18daf7b859da5b6ec3d5abaae1f3129d60899b3a16cf8b40381dc738d7

data/CHANGELOG.md

@@ -1,5 +1,27 @@
 # v1.10
 
+## Release v1.10.3 - 2020/05/01
+
+### Enhancement
+
+* record_accessor: Add `set` method
+  https://github.com/fluent/fluentd/pull/2977
+* config: Ruby DSL format is deprecated
+  https://github.com/fluent/fluentd/pull/2958
+* Refactor code
+  https://github.com/fluent/fluentd/pull/2961
+  https://github.com/fluent/fluentd/pull/2962
+  https://github.com/fluent/fluentd/pull/2965
+  https://github.com/fluent/fluentd/pull/2966
+  https://github.com/fluent/fluentd/pull/2978
+
+### Bug fix
+
+* out_forward: Disable `linger_timeout` setting on Windows
+  https://github.com/fluent/fluentd/pull/2959
+* out_forward: Fix warning of service discovery manager when fluentd stops
+  https://github.com/fluent/fluentd/pull/2974
+
 ## Release v1.10.2 - 2020/04/15
 
 ### Enhancement
@@ -59,7 +81,7 @@
 
 ### New feature
 
-* sd plugin: Add RSV record plugin
+* sd plugin: Add SRV record plugin
   https://github.com/fluent/fluentd/pull/2876
 
 ### Enhancement

data/lib/fluent/config.rb

@@ -62,6 +62,7 @@ module Fluent
         Parser.parse(str, fname, basepath)
       when :ruby
         require 'fluent/config/dsl'
+        $log.warn("Ruby DSL configuration format is deprecated. Please use original configuration format. https://docs.fluentd.org/configuration/config-file")
         Config::DSL::Parser.parse(str, File.join(basepath, fname))
       else
         raise "[BUG] unknown configuration parser specification:'#{parser}'"

data/lib/fluent/match.rb

@@ -125,7 +125,7 @@ module Fluent
     end
 
     def match(str)
-      @regex.match(str) != nil
+      @regex.match?(str)
     end
   end
 

data/lib/fluent/plugin/in_tail.rb

@@ -136,7 +136,7 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "#{rc} are reserved words: #{@path_delimiter}"
       end
 
-      @paths = @path.split(@path_delimiter).map(&:strip)
+      @paths = @path.split(@path_delimiter).map(&:strip).uniq
       if @paths.empty?
         raise Fluent::ConfigError, "tail: 'path' parameter is required on tail input"
       end
@@ -296,7 +296,7 @@ module Fluent::Plugin
                end
         path.include?('*') ? Dir.glob(path) : path
       }.flatten.uniq
-      paths - excluded
+      paths.uniq - excluded
     end
 
     # in_tail with '*' path doesn't check rotation file equality at refresh phase.

data/lib/fluent/plugin/out_forward.rb

@@ -382,7 +382,12 @@ module Fluent::Plugin
           cert_logical_store_name: @tls_cert_logical_store_name,
           cert_use_enterprise_store: @tls_cert_use_enterprise_store,
 
-          linger_timeout: @send_timeout,
+          # Enabling SO_LINGER causes tcp port exhaustion on Windows.
+          # This is because dynamic ports are only 16384 (from 49152 to 65535) and
+          # expiring SO_LINGER enabled ports should wait 4 minutes
+          # where set by TcpTimeDelay. Its default value is 4 minutes.
+          # So, we should disable SO_LINGER on Windows to prevent flood of waiting ports.
+          linger_timeout: Fluent.windows? ? nil : @send_timeout,
           send_timeout: @send_timeout,
           recv_timeout: @ack_response_timeout,
           connect_timeout: @connect_timeout,

data/lib/fluent/plugin/parser_multiline.rb

@@ -104,7 +104,7 @@ module Fluent
       end
 
       def firstline?(text)
-        @firstline_regex.match(text)
+        @firstline_regex.match?(text)
       end
 
       private

data/lib/fluent/plugin_helper/record_accessor.rb

@@ -41,9 +41,11 @@ module Fluent
             else
               mcall = method(:call_dig)
               mdelete = method(:delete_nest)
+              mset = method(:set_nest)
               singleton_class.module_eval do
                 define_method(:call, mcall)
                 define_method(:delete, mdelete)
+                define_method(:set, mset)
               end
             end
           end
@@ -75,6 +77,18 @@ module Fluent
           nil
         end
 
+        def set(r, v)
+          r[@keys] = v
+        end
+
+        # set_nest doesn't create intermediate object. If key doesn't exist, no effect.
+        # See also: https://bugs.ruby-lang.org/issues/11747
+        def set_nest(r, v)
+          r.dig(*@dig_keys)&.[]=(@last_key, v)
+        rescue
+          nil
+        end
+
         def self.parse_parameter(param)
           if param.start_with?('$.')
             parse_dot_notation(param)

data/lib/fluent/plugin_helper/service_discovery.rb

@@ -43,6 +43,13 @@ module Fluent
         super
       end
 
+      %i[after_start stop before_shutdown shutdown after_shutdown close terminate].each do |mth|
+        define_method(mth) do
+          @discovery_manager&.__send__(mth)
+          super()
+        end
+      end
+
       private
 
       # @param title [Symbol] the thread name. this value should be unique.

data/lib/fluent/plugin_helper/service_discovery/manager.rb

@@ -60,6 +60,14 @@ module Fluent
           end
         end
 
+        %i[after_start stop before_shutdown shutdown after_shutdown close terminate].each do |mth|
+          define_method(mth) do
+            @discoveries.each do |d|
+              d.__send__(mth)
+            end
+          end
+        end
+
         def run_once
           # Don't care race in this loop intentionally
           s = @queue.size

data/lib/fluent/plugin_helper/socket_option.rb

@@ -67,8 +67,8 @@ module Fluent
           end
         else
           if linger_timeout
-          optval = [1, linger_timeout.to_i].pack(FORMAT_STRUCT_LINGER)
-          socket_option_set_one(sock, :SO_LINGER, optval)
+            optval = [1, linger_timeout.to_i].pack(FORMAT_STRUCT_LINGER)
+            socket_option_set_one(sock, :SO_LINGER, optval)
           end
         end
         if recv_timeout

data/lib/fluent/supervisor.rb

@@ -556,7 +556,7 @@ module Fluent
       end
 
       if dry_run
-        $log.info 'finsihed dry run mode'
+        $log.info 'finished dry run mode'
         exit 0
       else
         supervise

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.10.2'
+  VERSION = '1.10.3'
 
 end

data/test/plugin/test_in_tail.rb

@@ -102,6 +102,12 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal ["tail.txt", "test2", "tmp,dev"], d.instance.paths
     end
 
+    test "multi paths with same path configured twice" do
+      c = config_element("ROOT", "", { "path" => "test1.txt,test2.txt,test1.txt", "tag" => "t1", "path_delimiter" => "," })
+      d = create_driver(c + PARSE_SINGLE_LINE_CONFIG, false)
+      assert_equal ["test2.txt","test1.txt"].sort, d.instance.paths.sort
+    end
+
     test "multi paths with invaid path_delimiter" do
       c = config_element("ROOT", "", { "path" => "tail.txt|test2|tmp,dev", "tag" => "t1", "path_delimiter" => "*" })
       assert_raise(Fluent::ConfigError) do
@@ -1022,6 +1028,17 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
     end
 
+    def test_expand_paths_with_duplicate_configuration
+      expanded_paths = [
+        'test/plugin/data/log/foo/bar.log',
+        'test/plugin/data/log/test.log'
+      ]
+      duplicate_config = EX_CONFIG.dup
+      duplicate_config["path"]="test/plugin/data/log/**/*.log, test/plugin/data/log/**/*.log"
+      plugin = create_driver(EX_CONFIG, false).instance
+      assert_equal expanded_paths, plugin.expand_paths.sort
+    end
+
     def test_expand_paths_with_timezone
       ['Asia/Taipei', '+08'].each do |tz_type|
         taipei_config = EX_CONFIG + config_element("", "", {"path_timezone" => tz_type})

data/test/plugin_helper/test_record_accessor.rb

@@ -194,4 +194,45 @@ class RecordAccessorHelperTest < Test::Unit::TestCase
       end
     end
   end
+
+  sub_test_case 'Fluent::PluginHelper::RecordAccessor::Accessor#set' do
+    setup do
+      @d = Dummy.new
+    end
+
+    data('normal' => 'key1',
+         'space' => 'ke y2',
+         'dot key' => 'this.is.key3')
+    test 'set top key' do |param|
+      r = {'key1' => 'v1', 'ke y2' => 'v2', 'this.is.key3' => 'v3'}
+      accessor = @d.record_accessor_create(param)
+      accessor.set(r, "test")
+      assert_equal "test", r[param]
+    end
+
+    test "set top key using bracket style" do
+      r = {'key1' => 'v1', 'ke y2' => 'v2', 'this.is.key3' => 'v3'}
+      accessor = @d.record_accessor_create('$["this.is.key3"]')
+      accessor.set(r, "test")
+      assert_equal "test", r["this.is.key3"]
+    end
+
+    data('bracket' => "$['key1'][0]['ke y2']",
+         'bracket w/ double quotes' => '$["key1"][0]["ke y2"]')
+    test "set nested keys ['key1', 0, 'ke y2']" do |param|
+      r = {'key1' => [{'ke y2' => "value"}]}
+      accessor = @d.record_accessor_create(param)
+      accessor.set(r, "nested_message")
+      assert_equal "nested_message", r['key1'][0]['ke y2']
+    end
+
+    test "don't raise an error when unexpected record is coming" do
+      r = {'key1' => [{'key3' => "value"}]}
+      accessor = @d.record_accessor_create("$['key1']['key2']['key3']")
+      assert_nothing_raised do
+        accessor.set(r, "unknown field")
+      end
+      assert_equal({'key1' => [{'key3' => "value"}]}, r)
+    end
+  end
 end

data/test/plugin_helper/test_service_discovery.rb

@@ -4,9 +4,6 @@ require 'fluent/plugin_helper/service_discovery'
 require 'fluent/plugin/output'
 
 class ServiceDiscoveryHelper < Test::Unit::TestCase
-  PORT = unused_port
-  NULL_LOGGER = Logger.new(nil)
-
   class Dummy < Fluent::Plugin::TestBase
     helpers :service_discovery
 
@@ -30,6 +27,7 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
     if @d
       @d.stop unless @d.stopped?
       @d.shutdown unless @d.shutdown?
+      @d.after_shutdown unless @d.after_shutdown?
       @d.close unless @d.closed?
       @d.terminate unless @d.terminated?
     end
@@ -40,7 +38,7 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
 
     d.service_discovery_create_manager(
       :service_discovery_helper_test,
-      configurations: [{ type: :static, conf:  config_element('root', '', {}, [config_element('service', '', { 'host' => '127.0.0.1', 'port' => '1234' })]) }],
+      configurations: [{ type: :static, conf: config_element('root', '', {}, [config_element('service', '', { 'host' => '127.0.0.1', 'port' => '1234' })]) }],
     )
 
     assert_true !!d.discovery_manager
@@ -49,6 +47,7 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
     mock.proxy(d).timer_execute(:service_discovery_helper_test, anything).never
 
     d.start
+    d.event_loop_wait_until_start
 
     services = d.discovery_manager.services
     assert_equal 1, services.size
@@ -68,5 +67,39 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
     mock.proxy(d.discovery_manager).start.once
     mock(d).timer_execute(:service_discovery_helper_test, anything).once
     d.start
+    d.event_loop_wait_until_start
+  end
+
+  test 'exits service discovery instances without any errors' do
+    d = @d = Dummy.new
+    mockv = flexmock('dns_resolver', getaddress: '127.0.0.1')
+              .should_receive(:getresources)
+              .and_return([Resolv::DNS::Resource::IN::SRV.new(1, 10, 8081, 'service1.example.com')])
+              .mock
+    mock(Resolv::DNS).new { mockv }
+
+    d.service_discovery_create_manager(
+      :service_discovery_helper_test2,
+      configurations: [{ type: :srv, conf: config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com' }) }],
+    )
+
+    assert_true !!d.discovery_manager
+    mock.proxy(d.discovery_manager).start.once
+    mock(d).timer_execute(:service_discovery_helper_test2, anything).once
+
+    # To avoid claring `@logs` during `terminate` step
+    # https://github.com/fluent/fluentd/blob/bc78d889f93dad8c2a4e0ad1ca802546185dacba/lib/fluent/test/log.rb#L33
+    mock(d.log).reset.twice
+
+    d.start
+    d.event_loop_wait_until_start
+
+    d.stop unless d.stopped?
+    d.shutdown unless d.shutdown?
+    d.after_shutdown unless d.after_shutdown?
+    d.close unless d.closed?
+    d.terminate unless d.terminated?
+
+    assert_false(d.log.out.logs.any? { |e| e.match?(/thread doesn't exit correctly/) })
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.10.2
+  version: 1.10.3
 platform: x86-mingw32
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-15 00:00:00.000000000 Z
+date: 2020-05-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack