fluentd 1.16.11-x86-mingw32 → 1.17.0-x86-mingw32

data/lib/fluent/plugin/parser_msgpack.rb

@@ -31,9 +31,9 @@ module Fluent
         :binary
       end
 
-      def parse(data)
+      def parse(data, &block)
         @unpacker.feed_each(data) do |obj|
-          yield convert_values(parse_time(obj), obj)
+          parse_unpacked_data(obj, &block)
         end
       end
       alias parse_partial_data parse
@@ -41,8 +41,29 @@ module Fluent
       def parse_io(io, &block)
         u = Fluent::MessagePackFactory.engine_factory.unpacker(io)
         u.each do |obj|
-          time, record = convert_values(parse_time(obj), obj)
+          parse_unpacked_data(obj, &block)
+        end
+      end
+
+      def parse_unpacked_data(data)
+        if data.is_a?(Hash)
+          time, record = convert_values(parse_time(data), data)
           yield time, record
+          return
+        end
+
+        unless data.is_a?(Array)
+          yield nil, nil
+          return
+        end
+
+        data.each do |record|
+          unless record.is_a?(Hash)
+            yield nil, nil
+            next
+          end
+          time, converted_record = convert_values(parse_time(record), record)
+          yield time, converted_record
         end
       end
     end

data/lib/fluent/plugin_helper/metrics.rb

@@ -65,9 +65,9 @@ module Fluent
         metrics.configure(config)
         # For multi workers environment, cmetrics should be distinguish with static labels.
         if Fluent::Engine.system_config.workers > 1
-          labels.merge!(worker_id: fluentd_worker_id.to_s)
+          labels[:worker_id] = fluentd_worker_id.to_s
         end
-        labels.merge!(plugin: @plugin_type_or_id)
+        labels[:plugin] = @plugin_type_or_id
         metrics.create(namespace: namespace, subsystem: subsystem, name: name, help_text: help_text, labels: labels)
 
         @_metrics["#{@plugin_type_or_id}_#{namespace}_#{subsystem}_#{name}"] = metrics

data/lib/fluent/plugin_helper/server.rb

@@ -347,10 +347,7 @@ module Fluent
       end
 
       def shutdown
-        # When it invokes conn.cose, it reduces elements in @_server_connections by close_callback,
-        # and it reduces the number of loops. This prevents the connection closing.
-        # So, it requires invoking #dup to avoid the problem.
-        @_server_connections.dup.each do |conn|
+        @_server_connections.each do |conn|
           conn.close rescue nil
         end
 

data/lib/fluent/registry.rb

@@ -60,13 +60,13 @@ module Fluent
       # search from additional plugin directories
       if @dir_search_prefix
         path = "#{@dir_search_prefix}#{type}"
-        files = @paths.map { |lp|
+        files = @paths.filter_map { |lp|
           lpath = File.expand_path(File.join(lp, "#{path}.rb"))
           File.exist?(lpath) ? lpath : nil
-        }.compact
+        }
         unless files.empty?
           # prefer newer version
-          require files.sort.last
+          require files.max
           return
         end
       end
@@ -74,17 +74,17 @@ module Fluent
       path = "#{@search_prefix}#{type}"
 
       # prefer LOAD_PATH than gems
-      files = $LOAD_PATH.map { |lp|
+      files = $LOAD_PATH.filter_map { |lp|
         if lp == FLUENT_LIB_PATH
           nil
         else
           lpath = File.expand_path(File.join(lp, "#{path}.rb"))
           File.exist?(lpath) ? lpath : nil
         end
-      }.compact
+      }
       unless files.empty?
         # prefer newer version
-        require files.sort.last
+        require files.max
         return
       end
 

data/lib/fluent/supervisor.rb

@@ -440,7 +440,7 @@ module Fluent
         stop_immediately_at_unrecoverable_exit: true,
         root_dir: params['root_dir'],
         logger: $log,
-        log: $log&.out,
+        log: $log.out,
         log_level: params['log_level'],
         chuser: params['chuser'],
         chgroup: params['chgroup'],
@@ -742,7 +742,7 @@ module Fluent
       $log.enable_debug if system_config.log_level <= Fluent::Log::LEVEL_DEBUG
 
       $log.info "init #{process_type} logger",
-                path: actual_log_path,
+                path: actual_log_path, 
                 rotate_age: @log_rotate_age,
                 rotate_size: @log_rotate_size
     end
@@ -995,7 +995,7 @@ module Fluent
     def build_system_config(conf)
       system_config = SystemConfig.create(conf, @cl_opt[:strict_config_value])
       # Prefer the options explicitly specified in the command line
-      #
+      # 
       # TODO: There is a bug that `system_config.log.rotate_age/rotate_size` are
       # not merged with the command line options since they are not in
       # `SYSTEM_CONFIG_PARAMETERS`.

data/lib/fluent/test/output_test.rb

@@ -139,7 +139,7 @@ module Fluent
             assert_equal(@expected_buffer, buffer)
           end
 
-          lines.keys.each do |meta|
+          lines.each_key do |meta|
             chunk = @instance.buffer.generate_chunk(meta).staged!
             chunk.append(lines[meta])
             begin

data/lib/fluent/test.rb

@@ -40,12 +40,9 @@ module Fluent
 
       $log = dummy_logger
 
-      old_engine = Fluent.__send__(:remove_const, :Engine)
-      # Ensure that GC can remove the objects of the old engine.
-      # Some objects can still exist after `remove_const`. See https://github.com/fluent/fluentd/issues/5054.
-      old_engine.instance_variable_set(:@root_agent, nil)
-
+      Fluent.__send__(:remove_const, :Engine)
       engine = Fluent.const_set(:Engine, EngineClass.new).init(SystemConfig.new)
+
       engine.define_singleton_method(:now=) {|n|
         @now = n
       }

data/lib/fluent/unique_id.rb

@@ -23,7 +23,7 @@ module Fluent
     end
 
     def self.hex(unique_id)
-      unique_id.unpack('H*').first
+      unique_id.unpack1('H*')
     end
 
     module Mixin

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.16.11'
+  VERSION = '1.17.0'
 
 end

data/lib/fluent/winsvc.rb

@@ -32,7 +32,7 @@ begin
   op.parse(ARGV)
   if opts[:service_name] == nil
     raise "Error: No Windows Service name set. Use '--service-name'"
-  end
+  end 
 
   def read_fluentdopt(service_name)
     Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\#{service_name}") do |reg|
@@ -48,32 +48,25 @@ begin
   end
 
   class FluentdService < Daemon
-    ERROR_WAIT_NO_CHILDREN = 128
-
     @pid = 0
     @service_name = ''
 
     def initialize(service_name)
       @service_name = service_name
     end
-
+    
     def service_main
       @pid = service_main_start(@service_name)
-      begin
-        loop do
-          sleep 5
-          break unless running?
-          raise Errno::ECHILD if Process.waitpid(@pid, Process::WNOHANG)
-        end
-      rescue Errno::ECHILD
-        @pid = 0
-        SetEvent(@@hStopEvent)
-        SetTheServiceStatus.call(SERVICE_STOPPED, ERROR_WAIT_NO_CHILDREN, 0, 0)
+      while running?
+        sleep 10
       end
     end
 
     def service_stop
-      wait_supervisor_finished if @pid > 0
+      set_event(@service_name)
+      if @pid > 0
+        Process.waitpid(@pid)
+      end
     end
 
     def service_paramchange
@@ -98,29 +91,6 @@ begin
       ev.set
       ev.close
     end
-
-    def repeat_set_event_several_times_until_success(event_name)
-      retries = 0
-      max_retries = 10
-      delay_sec = 3
-
-      begin
-        set_event(event_name)
-      rescue Errno::ENOENT
-        # This error occurs when the supervisor process has not yet created the event.
-        # If STOP is immediately executed, this state will occur.
-        # Retry `set_event' to wait for the initialization of the supervisor.
-        retries += 1
-        raise if max_retries < retries
-        sleep(delay_sec)
-        retry
-      end
-    end
-
-    def wait_supervisor_finished
-      repeat_set_event_several_times_until_success(@service_name)
-      Process.waitpid(@pid)
-    end
   end
 
   FluentdService.new(opts[:service_name]).mainloop

data/test/command/test_cat.rb

@@ -18,7 +18,7 @@ class TestFluentCat < ::Test::Unit::TestCase
     @primary = create_primary
     metadata = @primary.buffer.new_metadata
     @chunk = create_chunk(@primary, metadata, @es)
-    @port = unused_port(protocol: :all)
+    @port = unused_port
   end
 
   def teardown
@@ -87,7 +87,7 @@ class TestFluentCat < ::Test::Unit::TestCase
       d = create_driver
       d.run(expect_records: 1) do
         Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} --format msgpack secondary") do |stdin|
-          stdin.write(File.read(path, File.size(path)))
+          stdin.write(File.read(path))
           stdin.close
         end
       end

data/test/command/test_fluentd.rb

@@ -128,14 +128,11 @@ class TestFluentdCommand < ::Test::Unit::TestCase
 
   # ATTENTION: This stops taking logs when all `pattern_list` match or timeout,
   # so `patterns_not_match` can test only logs up to that point.
-  # You can pass a block to assert something after log matching.
   def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 20, env: {})
     matched = false
     matched_wrongly = false
-    error_msg_match = ""
+    assert_error_msg = ""
     stdio_buf = ""
-    succeeded_block = true
-    error_msg_block = ""
     begin
       execute_command(cmdline, @tmp_dir, env) do |pid, stdout|
         begin
@@ -166,13 +163,6 @@ class TestFluentdCommand < ::Test::Unit::TestCase
               end
             end
           end
-
-          begin
-            yield if block_given?
-          rescue => e
-            succeeded_block = false
-            error_msg_block = "failed block execution after matching: #{e}"
-          end
         ensure
           if SUPERVISOR_PID_PATTERN =~ stdio_buf
             @supervisor_pid = $1.to_i
@@ -183,19 +173,19 @@ class TestFluentdCommand < ::Test::Unit::TestCase
         end
       end
     rescue Timeout::Error
-      error_msg_match = "execution timeout"
+      assert_error_msg = "execution timeout"
       # https://github.com/fluent/fluentd/issues/4095
       # On Windows, timeout without `@supervisor_pid` means that the test is invalid,
       # since the supervisor process will survive without being killed correctly.
       flunk("Invalid test: The pid of supervisor could not be taken, which is necessary on Windows.") if Fluent.windows? && @supervisor_pid.nil?
     rescue => e
-      error_msg_match = "unexpected error in launching fluentd: #{e.inspect}"
+      assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}"
     else
-      error_msg_match = "log doesn't match" unless matched
+      assert_error_msg = "log doesn't match" unless matched
     end
 
     if patterns_not_match.empty?
-      error_msg_match = build_message(error_msg_match,
+      assert_error_msg = build_message(assert_error_msg,
                                        "<?>\nwas expected to include:\n<?>",
                                        stdio_buf, pattern_list)
     else
@@ -207,17 +197,16 @@ class TestFluentdCommand < ::Test::Unit::TestCase
                             lines.any?{|line| line.include?(ptn) }
                           end
         if matched_wrongly
-          error_msg_match << "\n" unless error_msg_match.empty?
-          error_msg_match << "pattern exists in logs wrongly: #{ptn}"
+          assert_error_msg << "\n" unless assert_error_msg.empty?
+          assert_error_msg << "pattern exists in logs wrongly: #{ptn}"
         end
       end
-      error_msg_match = build_message(error_msg_match,
+      assert_error_msg = build_message(assert_error_msg,
                                        "<?>\nwas expected to include:\n<?>\nand not include:\n<?>",
                                        stdio_buf, pattern_list, patterns_not_match)
     end
 
-    assert matched && !matched_wrongly, error_msg_match
-    assert succeeded_block, error_msg_block if block_given?
+    assert matched && !matched_wrongly, assert_error_msg
   end
 
   def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout: 20)
@@ -570,7 +559,7 @@ CONF
 
       assert_fluentd_fails_to_start(
         create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
-        /in_buggy.rb:\d+:.+\(SyntaxError\)/
+        "in_buggy.rb:5: syntax error, unexpected end-of-input"
       )
     end
   end
@@ -1175,7 +1164,7 @@ CONF
     end
   end
 
-  sub_test_case 'shared socket options' do
+  sub_test_case 'sahred socket options' do
     test 'enable shared socket by default' do
       conf = ""
       conf_path = create_conf_file('empty.conf', conf)
@@ -1299,40 +1288,4 @@ CONF
                          "[debug]")
     end
   end
-
-  sub_test_case "plugin option" do
-    test "should be the default value when not specifying" do
-      conf_path = create_conf_file('test.conf', <<~CONF)
-        <source>
-          @type monitor_agent
-        </source>
-      CONF
-      assert File.exist?(conf_path)
-      cmdline = create_cmdline(conf_path)
-
-      assert_log_matches(cmdline, "fluentd worker is now running") do
-        response = Net::HTTP.get(URI.parse("http://localhost:24220/api/config.json"))
-        actual_conf = JSON.parse(response)
-        assert_equal Fluent::Supervisor.default_options[:plugin_dirs], actual_conf["plugin_dirs"]
-      end
-    end
-
-    data(short: "-p")
-    data(long: "--plugin")
-    test "can be added by specifying the option" do |option_name|
-      conf_path = create_conf_file('test.conf', <<~CONF)
-        <source>
-          @type monitor_agent
-        </source>
-      CONF
-      assert File.exist?(conf_path)
-      cmdline = create_cmdline(conf_path, option_name, @tmp_dir, option_name, @tmp_dir)
-
-      assert_log_matches(cmdline, "fluentd worker is now running") do
-        response = Net::HTTP.get(URI.parse("http://localhost:24220/api/config.json"))
-        actual_conf = JSON.parse(response)
-        assert_equal Fluent::Supervisor.default_options[:plugin_dirs] + [@tmp_dir, @tmp_dir], actual_conf["plugin_dirs"]
-      end
-    end
-  end
 end

data/test/config/test_dsl.rb

@@ -1,7 +1,6 @@
 require_relative '../helper'
 require 'fluent/config/element'
 require "fluent/config/dsl"
-require 'tempfile'
 
 TMP_DIR = File.dirname(__FILE__) + "/tmp/config_dsl#{ENV['TEST_ENV_NUMBER']}"
 def write_config(path, data)
@@ -359,13 +358,9 @@ module Fluent::Config
       sub_test_case '.parse' do
         test 'can get result of Kernel.open() by ruby.open()' do
           uname_string = `uname -a`
-          tmpfile = Tempfile.create('fluentd-test')
-          tmpfile.write(uname_string)
-          tmpfile.close
-
           root = Fluent::Config::DSL::Parser.parse(<<DSL)
 worker {
-  uname_str = ruby.open("#{tmpfile.path}"){|out| out.read}
+  uname_str = ruby.open('|uname -a'){|out| out.read}
   source {
     uname uname_str
   }
@@ -377,8 +372,6 @@ DSL
           assert_equal('source', source.name)
           assert_equal(1, source.keys.size)
           assert_equal(uname_string, source['uname'])
-        ensure
-          File.delete(tmpfile.path)
         end
 
         test 'accepts ruby keyword with block, which allow to use methods included from ::Kernel' do

data/test/config/test_element.rb

@@ -282,8 +282,8 @@ CONF
       dump = <<-CONF
 name:ROOT, arg:, {\"k1\"=>\"v1\"}, [name:test, arg:ext, {\"k2\"=>\"v2\"}, []]
 CONF
-      assert_not_equal(e.to_s, e.inspect.gsub(' => ', '=>'))
-      assert_equal(dump.chomp, e.inspect.gsub(' => ', '=>'))
+      assert_not_equal(e.to_s, e.inspect)
+      assert_equal(dump.chomp, e.inspect)
     end
   end
 

data/test/config/test_plugin_configuration.rb

@@ -19,9 +19,9 @@ module ConfigurationForPlugins
 
   class BooleanParamsWithoutValue < ::Test::Unit::TestCase
     CONFIG = <<CONFIG
-    flag1
+    flag1 
     flag2 # yaaaaaaaaaay
-    flag3
+    flag3 
     flag4 # yaaaaaaaaaay
     <child>
       flag1
@@ -37,10 +37,10 @@ module ConfigurationForPlugins
     </child>
     # with following whitespace
     <child>
-      flag1
-      flag2
-      flag3
-      flag4
+      flag1 
+      flag2 
+      flag3 
+      flag4 
     </child>
 CONFIG
 

data/test/helper.rb

@@ -71,31 +71,17 @@ end
 
 include Fluent::Test::Helpers
 
-def unused_port(num = 1, protocol:, bind: "0.0.0.0")
+def unused_port(num = 1, protocol: :tcp, bind: "0.0.0.0")
   case protocol
-  when :tcp, :tls
+  when :tcp
     unused_port_tcp(num)
   when :udp
     unused_port_udp(num, bind: bind)
-  when :all
-    unused_port_tcp_udp(num)
   else
     raise ArgumentError, "unknown protocol: #{protocol}"
   end
 end
 
-def unused_port_tcp_udp(num = 1)
-  raise "not support num > 1" if num > 1
-
-  # The default maximum number of file descriptors in macOS is 256.
-  # It might need to set num to a smaller value than that.
-  tcp_ports = unused_port_tcp(200)
-  port = unused_port_udp(1, port_list: tcp_ports)
-  raise "can't find unused port" unless port
-
-  port
-end
-
 def unused_port_tcp(num = 1)
   ports = []
   sockets = []
@@ -104,7 +90,7 @@ def unused_port_tcp(num = 1)
     sockets << s
     ports << s.addr[1]
   end
-  sockets.each(&:close)
+  sockets.each{|s| s.close }
   if num == 1
     return ports.first
   else
@@ -114,15 +100,12 @@ end
 
 PORT_RANGE_AVAILABLE = (1024...65535)
 
-def unused_port_udp(num = 1, port_list: [], bind: "0.0.0.0")
+def unused_port_udp(num = 1, bind: "0.0.0.0")
   family = IPAddr.new(IPSocket.getaddress(bind)).ipv4? ? ::Socket::AF_INET : ::Socket::AF_INET6
   ports = []
   sockets = []
-
-  use_random_port = port_list.empty?
-  i = 0
-  loop do
-    port = use_random_port ? rand(PORT_RANGE_AVAILABLE) : port_list[i]
+  while ports.size < num
+    port = rand(PORT_RANGE_AVAILABLE)
     u = UDPSocket.new(family)
     if (u.bind(bind, port) rescue nil)
       ports << port
@@ -130,11 +113,8 @@ def unused_port_udp(num = 1, port_list: [], bind: "0.0.0.0")
     else
       u.close
     end
-    i += 1
-    break if ports.size >= num
-    break if !use_random_port && i >= port_list.size
   end
-  sockets.each(&:close)
+  sockets.each{|s| s.close }
   if num == 1
     return ports.first
   else

data/test/log/test_console_adapter.rb

@@ -72,11 +72,18 @@ class ConsoleAdapterTest < Test::Unit::TestCase
        fatal: :fatal)
   def test_options(level)
     @console_logger.send(level, "subject", kwarg1: "opt1", kwarg2: "opt2")
+    lines = @logdev.logs[0].split("\n")
+    args = JSON.load(lines[1..].collect { |str| str.sub(/\s+\|/, "") }.join("\n"));
     assert_equal([
-                   "#{@timestamp_str} [#{level}]:   0.0s: subject\n" +
-                   "      | {\"kwarg1\":\"opt1\",\"kwarg2\":\"opt2\"}\n"
+                   1,
+                   "#{@timestamp_str} [#{level}]:   0.0s: subject",
+                   { "kwarg1" => "opt1", "kwarg2" => "opt2" }
                  ],
-                 @logdev.logs)
+                 [
+                   @logdev.logs.size,
+                   lines[0],
+                   args
+                 ])
   end
 
   data(debug: :debug,

data/test/plugin/in_tail/test_io_handler.rb

@@ -5,19 +5,20 @@ require 'fluent/plugin/metrics_local'
 require 'tempfile'
 
 class IntailIOHandlerTest < Test::Unit::TestCase
-  def setup
-    Tempfile.create('intail_io_handler') do |file|
-      file.binmode
-      @file = file
-      opened_file_metrics = Fluent::Plugin::LocalMetrics.new
-      opened_file_metrics.configure(config_element('metrics', '', {}))
-      closed_file_metrics = Fluent::Plugin::LocalMetrics.new
-      closed_file_metrics.configure(config_element('metrics', '', {}))
-      rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
-      rotated_file_metrics.configure(config_element('metrics', '', {}))
-      @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
-      yield
-    end
+  setup do
+    @file = Tempfile.new('intail_io_handler').binmode
+    opened_file_metrics = Fluent::Plugin::LocalMetrics.new
+    opened_file_metrics.configure(config_element('metrics', '', {}))
+    closed_file_metrics = Fluent::Plugin::LocalMetrics.new
+    closed_file_metrics.configure(config_element('metrics', '', {}))
+    rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
+    rotated_file_metrics.configure(config_element('metrics', '', {}))
+    @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
+  end
+
+  teardown do
+    @file.close rescue nil
+    @file.unlink rescue nil
   end
 
   def create_target_info

data/test/plugin/in_tail/test_position_file.rb

@@ -6,12 +6,13 @@ require 'fileutils'
 require 'tempfile'
 
 class IntailPositionFileTest < Test::Unit::TestCase
-  def setup
-    Tempfile.create('intail_position_file_test') do |file|
-      file.binmode
-      @file = file
-      yield
-    end
+  setup do
+    @file = Tempfile.new('intail_position_file_test').binmode
+  end
+
+  teardown do
+    @file.close rescue nil
+    @file.unlink rescue nil
   end
 
   UNWATCHED_STR = '%016x' % Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION

data/test/plugin/out_forward/test_ack_handler.rb

@@ -111,11 +111,11 @@ class AckHandlerTest < Test::Unit::TestCase
     r, w = IO.pipe
     begin
       w.write(chunk_id)
-      def r.recv(arg)
+      stub(r).recv { |_|
         sleep(1) # To ensure that multiple threads select the socket before closing.
-        raise IOError, 'stream closed in another thread' if self.closed?
+        raise IOError, 'stream closed in another thread' if r.closed?
         MessagePack.pack({ 'ack' => Base64.encode64('chunk_id 111') })
-      end
+      }
       ack.enqueue(r)
 
       threads = []