fluentd 1.16.8-x86-mingw32 → 1.17.0-x86-mingw32

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.16.8'
+  VERSION = '1.17.0'
 
 end

data/lib/fluent/winsvc.rb

@@ -32,7 +32,7 @@ begin
   op.parse(ARGV)
   if opts[:service_name] == nil
     raise "Error: No Windows Service name set. Use '--service-name'"
-  end
+  end 
 
   def read_fluentdopt(service_name)
     Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\#{service_name}") do |reg|
@@ -48,32 +48,25 @@ begin
   end
 
   class FluentdService < Daemon
-    ERROR_WAIT_NO_CHILDREN = 128
-
     @pid = 0
     @service_name = ''
 
     def initialize(service_name)
       @service_name = service_name
     end
-
+    
     def service_main
       @pid = service_main_start(@service_name)
-      begin
-        loop do
-          sleep 5
-          break unless running?
-          raise Errno::ECHILD unless Process.waitpid2(@pid, Process::WNOHANG)
-        end
-      rescue Errno::ECHILD
-        @pid = 0
-        SetEvent(@@hStopEvent)
-        SetTheServiceStatus.call(SERVICE_STOPPED, ERROR_WAIT_NO_CHILDREN, 0, 0)
+      while running?
+        sleep 10
       end
     end
 
     def service_stop
-      wait_supervisor_finished if @pid > 0
+      set_event(@service_name)
+      if @pid > 0
+        Process.waitpid(@pid)
+      end
     end
 
     def service_paramchange
@@ -98,29 +91,6 @@ begin
       ev.set
       ev.close
     end
-
-    def repeat_set_event_several_times_until_success(event_name)
-      retries = 0
-      max_retries = 10
-      delay_sec = 3
-
-      begin
-        set_event(event_name)
-      rescue Errno::ENOENT
-        # This error occurs when the supervisor process has not yet created the event.
-        # If STOP is immediately executed, this state will occur.
-        # Retry `set_event' to wait for the initialization of the supervisor.
-        retries += 1
-        raise if max_retries < retries
-        sleep(delay_sec)
-        retry
-      end
-    end
-
-    def wait_supervisor_finished
-      repeat_set_event_several_times_until_success(@service_name)
-      Process.waitpid(@pid)
-    end
   end
 
   FluentdService.new(opts[:service_name]).mainloop

data/test/command/test_cat.rb

@@ -18,7 +18,7 @@ class TestFluentCat < ::Test::Unit::TestCase
     @primary = create_primary
     metadata = @primary.buffer.new_metadata
     @chunk = create_chunk(@primary, metadata, @es)
-    @port = unused_port(protocol: :all)
+    @port = unused_port
   end
 
   def teardown
@@ -87,7 +87,7 @@ class TestFluentCat < ::Test::Unit::TestCase
       d = create_driver
       d.run(expect_records: 1) do
         Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} --format msgpack secondary") do |stdin|
-          stdin.write(File.read(path, File.size(path)))
+          stdin.write(File.read(path))
           stdin.close
         end
       end

data/test/command/test_fluentd.rb

@@ -128,14 +128,11 @@ class TestFluentdCommand < ::Test::Unit::TestCase
 
   # ATTENTION: This stops taking logs when all `pattern_list` match or timeout,
   # so `patterns_not_match` can test only logs up to that point.
-  # You can pass a block to assert something after log matching.
   def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 20, env: {})
     matched = false
     matched_wrongly = false
-    error_msg_match = ""
+    assert_error_msg = ""
     stdio_buf = ""
-    succeeded_block = true
-    error_msg_block = ""
     begin
       execute_command(cmdline, @tmp_dir, env) do |pid, stdout|
         begin
@@ -166,13 +163,6 @@ class TestFluentdCommand < ::Test::Unit::TestCase
               end
             end
           end
-
-          begin
-            yield if block_given?
-          rescue => e
-            succeeded_block = false
-            error_msg_block = "failed block execution after matching: #{e}"
-          end
         ensure
           if SUPERVISOR_PID_PATTERN =~ stdio_buf
             @supervisor_pid = $1.to_i
@@ -183,19 +173,19 @@ class TestFluentdCommand < ::Test::Unit::TestCase
         end
       end
     rescue Timeout::Error
-      error_msg_match = "execution timeout"
+      assert_error_msg = "execution timeout"
       # https://github.com/fluent/fluentd/issues/4095
       # On Windows, timeout without `@supervisor_pid` means that the test is invalid,
       # since the supervisor process will survive without being killed correctly.
       flunk("Invalid test: The pid of supervisor could not be taken, which is necessary on Windows.") if Fluent.windows? && @supervisor_pid.nil?
     rescue => e
-      error_msg_match = "unexpected error in launching fluentd: #{e.inspect}"
+      assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}"
     else
-      error_msg_match = "log doesn't match" unless matched
+      assert_error_msg = "log doesn't match" unless matched
     end
 
     if patterns_not_match.empty?
-      error_msg_match = build_message(error_msg_match,
+      assert_error_msg = build_message(assert_error_msg,
                                        "<?>\nwas expected to include:\n<?>",
                                        stdio_buf, pattern_list)
     else
@@ -207,17 +197,16 @@ class TestFluentdCommand < ::Test::Unit::TestCase
                             lines.any?{|line| line.include?(ptn) }
                           end
         if matched_wrongly
-          error_msg_match << "\n" unless error_msg_match.empty?
-          error_msg_match << "pattern exists in logs wrongly: #{ptn}"
+          assert_error_msg << "\n" unless assert_error_msg.empty?
+          assert_error_msg << "pattern exists in logs wrongly: #{ptn}"
         end
       end
-      error_msg_match = build_message(error_msg_match,
+      assert_error_msg = build_message(assert_error_msg,
                                        "<?>\nwas expected to include:\n<?>\nand not include:\n<?>",
                                        stdio_buf, pattern_list, patterns_not_match)
     end
 
-    assert matched && !matched_wrongly, error_msg_match
-    assert succeeded_block, error_msg_block if block_given?
+    assert matched && !matched_wrongly, assert_error_msg
   end
 
   def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout: 20)
@@ -1175,7 +1164,7 @@ CONF
     end
   end
 
-  sub_test_case 'shared socket options' do
+  sub_test_case 'sahred socket options' do
     test 'enable shared socket by default' do
       conf = ""
       conf_path = create_conf_file('empty.conf', conf)
@@ -1299,40 +1288,4 @@ CONF
                          "[debug]")
     end
   end
-
-  sub_test_case "plugin option" do
-    test "should be the default value when not specifying" do
-      conf_path = create_conf_file('test.conf', <<~CONF)
-        <source>
-          @type monitor_agent
-        </source>
-      CONF
-      assert File.exist?(conf_path)
-      cmdline = create_cmdline(conf_path)
-
-      assert_log_matches(cmdline, "fluentd worker is now running") do
-        response = Net::HTTP.get(URI.parse("http://localhost:24220/api/config.json"))
-        actual_conf = JSON.parse(response)
-        assert_equal Fluent::Supervisor.default_options[:plugin_dirs], actual_conf["plugin_dirs"]
-      end
-    end
-
-    data(short: "-p")
-    data(long: "--plugin")
-    test "can be added by specifying the option" do |option_name|
-      conf_path = create_conf_file('test.conf', <<~CONF)
-        <source>
-          @type monitor_agent
-        </source>
-      CONF
-      assert File.exist?(conf_path)
-      cmdline = create_cmdline(conf_path, option_name, @tmp_dir, option_name, @tmp_dir)
-
-      assert_log_matches(cmdline, "fluentd worker is now running") do
-        response = Net::HTTP.get(URI.parse("http://localhost:24220/api/config.json"))
-        actual_conf = JSON.parse(response)
-        assert_equal Fluent::Supervisor.default_options[:plugin_dirs] + [@tmp_dir, @tmp_dir], actual_conf["plugin_dirs"]
-      end
-    end
-  end
 end

data/test/config/test_plugin_configuration.rb

@@ -19,9 +19,9 @@ module ConfigurationForPlugins
 
   class BooleanParamsWithoutValue < ::Test::Unit::TestCase
     CONFIG = <<CONFIG
-    flag1
+    flag1 
     flag2 # yaaaaaaaaaay
-    flag3
+    flag3 
     flag4 # yaaaaaaaaaay
     <child>
       flag1
@@ -37,10 +37,10 @@ module ConfigurationForPlugins
     </child>
     # with following whitespace
     <child>
-      flag1
-      flag2
-      flag3
-      flag4
+      flag1 
+      flag2 
+      flag3 
+      flag4 
     </child>
 CONFIG
 

data/test/helper.rb

@@ -71,31 +71,17 @@ end
 
 include Fluent::Test::Helpers
 
-def unused_port(num = 1, protocol:, bind: "0.0.0.0")
+def unused_port(num = 1, protocol: :tcp, bind: "0.0.0.0")
   case protocol
-  when :tcp, :tls
+  when :tcp
     unused_port_tcp(num)
   when :udp
     unused_port_udp(num, bind: bind)
-  when :all
-    unused_port_tcp_udp(num)
   else
     raise ArgumentError, "unknown protocol: #{protocol}"
   end
 end
 
-def unused_port_tcp_udp(num = 1)
-  raise "not support num > 1" if num > 1
-
-  # The default maximum number of file descriptors in macOS is 256.
-  # It might need to set num to a smaller value than that.
-  tcp_ports = unused_port_tcp(200)
-  port = unused_port_udp(1, port_list: tcp_ports)
-  raise "can't find unused port" unless port
-
-  port
-end
-
 def unused_port_tcp(num = 1)
   ports = []
   sockets = []
@@ -104,7 +90,7 @@ def unused_port_tcp(num = 1)
     sockets << s
     ports << s.addr[1]
   end
-  sockets.each(&:close)
+  sockets.each{|s| s.close }
   if num == 1
     return ports.first
   else
@@ -114,15 +100,12 @@ end
 
 PORT_RANGE_AVAILABLE = (1024...65535)
 
-def unused_port_udp(num = 1, port_list: [], bind: "0.0.0.0")
+def unused_port_udp(num = 1, bind: "0.0.0.0")
   family = IPAddr.new(IPSocket.getaddress(bind)).ipv4? ? ::Socket::AF_INET : ::Socket::AF_INET6
   ports = []
   sockets = []
-
-  use_random_port = port_list.empty?
-  i = 0
-  loop do
-    port = use_random_port ? rand(PORT_RANGE_AVAILABLE) : port_list[i]
+  while ports.size < num
+    port = rand(PORT_RANGE_AVAILABLE)
     u = UDPSocket.new(family)
     if (u.bind(bind, port) rescue nil)
       ports << port
@@ -130,11 +113,8 @@ def unused_port_udp(num = 1, port_list: [], bind: "0.0.0.0")
     else
       u.close
     end
-    i += 1
-    break if ports.size >= num
-    break if !use_random_port && i >= port_list.size
   end
-  sockets.each(&:close)
+  sockets.each{|s| s.close }
   if num == 1
     return ports.first
   else

data/test/log/test_console_adapter.rb

@@ -72,11 +72,18 @@ class ConsoleAdapterTest < Test::Unit::TestCase
        fatal: :fatal)
   def test_options(level)
     @console_logger.send(level, "subject", kwarg1: "opt1", kwarg2: "opt2")
+    lines = @logdev.logs[0].split("\n")
+    args = JSON.load(lines[1..].collect { |str| str.sub(/\s+\|/, "") }.join("\n"));
     assert_equal([
-                   "#{@timestamp_str} [#{level}]:   0.0s: subject\n" +
-                   "      | {\"kwarg1\":\"opt1\",\"kwarg2\":\"opt2\"}\n"
+                   1,
+                   "#{@timestamp_str} [#{level}]:   0.0s: subject",
+                   { "kwarg1" => "opt1", "kwarg2" => "opt2" }
                  ],
-                 @logdev.logs)
+                 [
+                   @logdev.logs.size,
+                   lines[0],
+                   args
+                 ])
   end
 
   data(debug: :debug,

data/test/plugin/in_tail/test_io_handler.rb

@@ -5,19 +5,20 @@ require 'fluent/plugin/metrics_local'
 require 'tempfile'
 
 class IntailIOHandlerTest < Test::Unit::TestCase
-  def setup
-    Tempfile.create('intail_io_handler') do |file|
-      file.binmode
-      @file = file
-      opened_file_metrics = Fluent::Plugin::LocalMetrics.new
-      opened_file_metrics.configure(config_element('metrics', '', {}))
-      closed_file_metrics = Fluent::Plugin::LocalMetrics.new
-      closed_file_metrics.configure(config_element('metrics', '', {}))
-      rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
-      rotated_file_metrics.configure(config_element('metrics', '', {}))
-      @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
-      yield
-    end
+  setup do
+    @file = Tempfile.new('intail_io_handler').binmode
+    opened_file_metrics = Fluent::Plugin::LocalMetrics.new
+    opened_file_metrics.configure(config_element('metrics', '', {}))
+    closed_file_metrics = Fluent::Plugin::LocalMetrics.new
+    closed_file_metrics.configure(config_element('metrics', '', {}))
+    rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
+    rotated_file_metrics.configure(config_element('metrics', '', {}))
+    @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
+  end
+
+  teardown do
+    @file.close rescue nil
+    @file.unlink rescue nil
   end
 
   def create_target_info

data/test/plugin/in_tail/test_position_file.rb

@@ -6,12 +6,13 @@ require 'fileutils'
 require 'tempfile'
 
 class IntailPositionFileTest < Test::Unit::TestCase
-  def setup
-    Tempfile.create('intail_position_file_test') do |file|
-      file.binmode
-      @file = file
-      yield
-    end
+  setup do
+    @file = Tempfile.new('intail_position_file_test').binmode
+  end
+
+  teardown do
+    @file.close rescue nil
+    @file.unlink rescue nil
   end
 
   UNWATCHED_STR = '%016x' % Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION

data/test/plugin/out_forward/test_ack_handler.rb

@@ -111,11 +111,11 @@ class AckHandlerTest < Test::Unit::TestCase
     r, w = IO.pipe
     begin
       w.write(chunk_id)
-      def r.recv(arg)
+      stub(r).recv { |_|
         sleep(1) # To ensure that multiple threads select the socket before closing.
-        raise IOError, 'stream closed in another thread' if self.closed?
+        raise IOError, 'stream closed in another thread' if r.closed?
         MessagePack.pack({ 'ack' => Base64.encode64('chunk_id 111') })
-      end
+      }
       ack.enqueue(r)
 
       threads = []

data/test/plugin/out_forward/test_socket_cache.rb

@@ -17,7 +17,7 @@ class SocketCacheTest < Test::Unit::TestCase
       assert_equal(socket, c.checkout_or('key') { socket })
       c.checkin(socket)
 
-      sock = mock!.open.never.subject
+      sock = dont_allow(mock!).open
       assert_equal(socket, c.checkout_or('key') { sock.open })
     end
 
@@ -130,7 +130,7 @@ class SocketCacheTest < Test::Unit::TestCase
 
       c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
       sock = mock!.close { 'closed' }.subject
-      sock2 = mock!.close.never.subject
+      sock2 = dont_allow(mock!).close
       stub(sock).inspect
       stub(sock2).inspect
 
@@ -154,7 +154,7 @@ class SocketCacheTest < Test::Unit::TestCase
       Timecop.freeze(Time.parse('2016-04-13 14:00:00 +0900'))
 
       c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
-      sock = mock!.close.never.subject
+      sock = dont_allow(mock!).close
       stub(sock).inspect
       c.checkout_or('key') { sock }
 

data/test/plugin/test_buffer.rb

@@ -946,9 +946,9 @@ class BufferTest < Test::Unit::TestCase
       # 1. `write_once`: 42 [events] * 1 [stream]
       # 2. `write_step_by_step`: 4 [events]* 10 [streams] + 2 [events] * 1 [stream]
       # 3. `write_step_by_step` (by `ShouldRetry`): 1 [event] * 42 [streams]
-      #
+      # 
       # Example of staged chunk lock behavior:
-      #
+      # 
       # 1. mon_enter in write_step_by_step
       # 2. ShouldRetry occurs
       # 3. mon_exit in write_step_by_step

data/test/plugin/test_filter_grep.rb

@@ -328,7 +328,7 @@ class GrepFilterTest < Test::Unit::TestCase
       end
 
       test "don't raise an exception" do
-        assert_nothing_raised {
+        assert_nothing_raised { 
           filter(%[regexp1 message WARN], ["\xff".force_encoding('UTF-8')])
         }
       end

data/test/plugin/test_in_forward.rb

@@ -18,8 +18,7 @@ class ForwardInputTest < Test::Unit::TestCase
     Fluent::Test.setup
     @responses = []  # for testing responses after sending data
     @d = nil
-    # forward plugin uses TCP and UDP sockets on the same port number
-    @port = unused_port(protocol: :all)
+    @port = unused_port
   end
 
   def teardown

data/test/plugin/test_in_http.rb

@@ -18,7 +18,7 @@ class HttpInputTest < Test::Unit::TestCase
 
   def setup
     Fluent::Test.setup
-    @port = unused_port(protocol: :tcp)
+    @port = unused_port
   end
 
   def teardown
@@ -517,6 +517,28 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[1][1]
   end
 
+  def test_csp_report
+    d = create_driver
+    time = event_time("2011-01-02 13:14:15 UTC")
+    time_i = time.to_i
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag2", time, {"a"=>2}],
+    ]
+    res_codes = []
+
+    d.run(expect_records: 2) do
+      events.each do |tag, t, record|
+        res = post("/#{tag}?time=#{time_i.to_s}", record.to_json, {"Content-Type"=>"application/csp-report; charset=utf-8"})
+        res_codes << res.code
+      end
+    end
+    assert_equal ["200", "200"], res_codes
+    assert_equal events, d.events
+    assert_equal_event_time time, d.events[0][1]
+    assert_equal_event_time time, d.events[1][1]
+  end
+
   def test_application_msgpack
     d = create_driver
     time = event_time("2011-01-02 13:14:15 UTC")

data/test/plugin/test_in_monitor_agent.rb

@@ -392,7 +392,7 @@ EOC
     end
 
     test "emit" do
-      port = unused_port(protocol: :tcp)
+      port = unused_port
       d = create_driver("
   @type monitor_agent
   bind '127.0.0.1'
@@ -451,7 +451,7 @@ EOC
 
   sub_test_case "servlets" do
     setup do
-      @port = unused_port(protocol: :tcp)
+      @port = unused_port
       # check @type and type in one configuration
       conf = <<-EOC
 <source>
@@ -759,7 +759,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
     end
 
     setup do
-      @port = unused_port(protocol: :tcp)
+      @port = unused_port
       # check @type and type in one configuration
       conf = <<-EOC
 <source>
@@ -840,7 +840,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
 
   sub_test_case "check the port number of http server" do
     test "on single worker environment" do
-      port = unused_port(protocol: :tcp)
+      port = unused_port
       d = create_driver("
   @type monitor_agent
   bind '127.0.0.1'
@@ -851,7 +851,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
     end
 
     test "worker_id = 2 on multi worker environment" do
-      port = unused_port(protocol: :tcp)
+      port = unused_port
       Fluent::SystemConfig.overwrite_system_config('workers' => 4) do
         d = Fluent::Test::Driver::Input.new(Fluent::Plugin::MonitorAgentInput)
         d.instance.instance_eval{ @_fluentd_worker_id = 2 }
@@ -905,7 +905,7 @@ EOC
     end
 
     test "plugins have a variable named buffer does not throws NoMethodError" do
-      port = unused_port(protocol: :tcp)
+      port = unused_port
       d = create_driver("
   @type monitor_agent
   bind '127.0.0.1'