fluentd 1.16.6-x64-mingw-ucrt → 1.16.8-x64-mingw-ucrt

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8da42c71a4e549e36487a1c2aa44f6f3c57e334de07b2db530bb8b53c8e2c234
-  data.tar.gz: d7953b063d58d20cbc3b8b7bbddc496f6682add58fce983bfa5f24e1b33824e5
+  metadata.gz: 4afb099e933cca84ae72dbca686fb1c9e22b029f2f3ef08117148fd926bd3fe6
+  data.tar.gz: 365f948071b7e8da1c90e8b398c20612859def3a3eea68b095ac862565dc7940
 SHA512:
-  metadata.gz: b832e44c501295a88e89d2a5ca47c2846293252101ad9dd7a79f93a381344201c7eb2cb4578179a19f5405a22d23eab2d0f3f41f3e04823d6729d4b5483bbcfa
-  data.tar.gz: 5c7bf91228f746fdc17ec24dfb2d54e62a58853c8e55a0b58e2d1652061f496da9a07ba9019f36084e42f724d426f570e9d648bceba26a379b98858bcdd0f291
+  metadata.gz: ccf7f9f7034887aef88dafcc770417241f50d657ed2cdef8083502fe533607caf9449acb00b67a3d450e081bd4205432abf55599a3a443eb04f96f1787680e62
+  data.tar.gz: e41d5ac1095ce23bae510d9ca87804e13872ccb53bfd229a9d28668556c8aef8c1c01a4a744ea414108dcf9bd486e10be38a29339327451ffcfdae7e749aae53

data/.github/workflows/test.yml

@@ -12,6 +12,10 @@ on:
       - '*.md'
       - 'lib/fluent/version.rb'
 
+concurrency:
+  group: ${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ${{ matrix.os }}
@@ -29,6 +33,7 @@ jobs:
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby-version }}
+          rubygems: latest
       - name: Install addons
         if: ${{ matrix.os == 'ubuntu-latest' }}
         run: sudo apt-get install libgmp3-dev libcap-ng-dev

data/CHANGELOG.md

@@ -1,5 +1,65 @@
 # v1.16
 
+## Release v1.16.8 - 2025/05/01
+
+### Bug Fix
+
+* winsvc: Stop the service when the supervisor is dead
+  https://github.com/fluent/fluentd/pull/4942
+* formatter_csv: Fix memory leak
+  https://github.com/fluent/fluentd/pull/4920
+
+### Misc
+
+* Add fiddle as dependency gem for Ruby 3.5 on Windows
+  https://github.com/fluent/fluentd/pull/4919
+* Refactoring code
+  https://github.com/fluent/fluentd/pull/4921
+  https://github.com/fluent/fluentd/pull/4922
+  https://github.com/fluent/fluentd/pull/4926
+  https://github.com/fluent/fluentd/pull/4943
+* CI improvemnts
+  https://github.com/fluent/fluentd/pull/4821
+  https://github.com/fluent/fluentd/pull/4850
+  https://github.com/fluent/fluentd/pull/4851
+  https://github.com/fluent/fluentd/pull/4862
+  https://github.com/fluent/fluentd/pull/4915
+  https://github.com/fluent/fluentd/pull/4923
+  https://github.com/fluent/fluentd/pull/4925
+  https://github.com/fluent/fluentd/pull/4927
+
+## Release v1.16.7 - 2025/01/29
+
+### Bug Fix
+
+* Windows: Fix `NoMethodError` of `--daemon` option
+  https://github.com/fluent/fluentd/pull/4796
+* Windows: Fixed an issue where stopping the service immediately after startup could leave the processes
+  https://github.com/fluent/fluentd/pull/4782
+* Windows: Fixed an issue where stopping service sometimes can not be completed forever
+  https://github.com/fluent/fluentd/pull/4782
+
+### Misc
+
+* Windows: Add workaround for unexpected exception
+  https://github.com/fluent/fluentd/pull/4747
+* README: remove deprecated google analytics beacon
+  https://github.com/fluent/fluentd/pull/4797
+* CI improvemnts
+  https://github.com/fluent/fluentd/pull/4723
+  https://github.com/fluent/fluentd/pull/4788
+  https://github.com/fluent/fluentd/pull/4789
+  https://github.com/fluent/fluentd/pull/4790
+  https://github.com/fluent/fluentd/pull/4791
+  https://github.com/fluent/fluentd/pull/4793
+  https://github.com/fluent/fluentd/pull/4794
+  https://github.com/fluent/fluentd/pull/4795
+  https://github.com/fluent/fluentd/pull/4798
+  https://github.com/fluent/fluentd/pull/4799
+  https://github.com/fluent/fluentd/pull/4800
+  https://github.com/fluent/fluentd/pull/4801
+  https://github.com/fluent/fluentd/pull/4803
+
 ## Release v1.16.6 - 2024/08/16
 
 ### Bug Fix

data/README.md

@@ -71,5 +71,3 @@ See [SECURITY](SECURITY.md) to contact us about vulnerability.
 ## Contributors:
 
 Patches contributed by [great developers](https://github.com/fluent/fluentd/contributors).
-
-[<img src="https://ga-beacon.appspot.com/UA-24890265-6/fluent/fluentd" />](https://github.com/fluent/fluentd)

data/fluentd.gemspec

@@ -31,6 +31,9 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("webrick", ["~> 1.4"])
   gem.add_runtime_dependency("console", ["< 1.24"])
 
+  # gems that aren't default gems as of Ruby 3.5
+  gem.add_runtime_dependency("logger", ["~> 1.6"])
+
   # build gem for a certain platform. see also Rakefile
   fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
   gem.platform = fake_platform unless fake_platform.empty?
@@ -39,6 +42,9 @@ Gem::Specification.new do |gem|
     gem.add_runtime_dependency("win32-ipc", ["~> 0.7.0"])
     gem.add_runtime_dependency("win32-event", ["~> 0.6.3"])
     gem.add_runtime_dependency("certstore_c", ["~> 0.1.7"])
+
+    # gems that aren't default gems as of Ruby 3.5
+    gem.add_runtime_dependency("fiddle", ["~> 1.1"])
   end
 
   gem.add_development_dependency("rake", ["~> 13.0"])

data/lib/fluent/compat/formatter.rb

@@ -101,6 +101,12 @@ module Fluent
 
       class CsvFormatter < Fluent::Plugin::CsvFormatter
         # TODO: warn when deprecated
+
+        # Do not cache because it is hard to consider the thread key correctly.
+        # (We can try, but it would be low priority.)
+        def csv_cacheable?
+          false
+        end
       end
 
       class SingleValueFormatter < Fluent::Plugin::SingleValueFormatter

data/lib/fluent/config/parser.rb

@@ -92,10 +92,23 @@ module Fluent
         else
           basepath = '/'
           fname = path
-          require 'open-uri'
-          URI.open(uri) {|f|
+          parser_proc = ->(f) {
             Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
           }
+
+          case u.scheme
+          when 'http', 'https', 'ftp'
+            # URI#open can be able to handle URIs for http, https and ftp.
+            require 'open-uri'
+            u.open(&parser_proc)
+          else
+            # TODO: This case should be handled in the previous if condition. Glob is not applied to some Windows path formats.
+            # 'c:/path/to/file' will be passed as URI, 'uri' and 'u.path' will be:
+            #   - uri is 'c:/path/to/file'
+            #   - u.path is '/path/to/file' and u.scheme is 'c'
+            # Therefore, the condition of the if statement above is not met and it is handled here.
+            File.open(uri, &parser_proc)
+          end
         end
 
       rescue SystemCallError => e
@@ -104,4 +117,3 @@ module Fluent
     end
   end
 end
-

data/lib/fluent/config/v1_parser.rb

@@ -172,7 +172,7 @@ module Fluent
           require 'open-uri'
           basepath = '/'
           fname = path
-          data = URI.open(uri) { |f| f.read }
+          data = u.open { |f| f.read }
           data.force_encoding('UTF-8')
           ss = StringScanner.new(data)
           V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)

data/lib/fluent/plugin/formatter_csv.rb

@@ -35,6 +35,22 @@ module Fluent
       config_param :fields, :array, value_type: :string
       config_param :add_newline, :bool, default: true
 
+      def csv_cacheable?
+        !!owner
+      end
+
+      def csv_thread_key
+        csv_cacheable? ? "#{owner.plugin_id}_csv_formatter_#{@usage}_csv" : nil
+      end
+
+      def csv_for_thread
+        if csv_cacheable?
+          Thread.current[csv_thread_key] ||= CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts)
+        else
+          CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts)
+        end
+      end
+
       def configure(conf)
         super
 
@@ -51,12 +67,10 @@ module Fluent
 
         @generate_opts = {col_sep: @delimiter, force_quotes: @force_quotes, headers: @fields,
                           row_sep: @add_newline ? :auto : "".force_encoding(Encoding::ASCII_8BIT)}
-        # Cache CSV object per thread to avoid internal state sharing
-        @cache = {}
       end
 
       def format(tag, time, record)
-        csv = (@cache[Thread.current] ||= CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts))
+        csv = csv_for_thread
         line = (csv << record).string.dup
         # Need manual cleanup because CSV writer doesn't provide such method.
         csv.rewind
@@ -65,7 +79,7 @@ module Fluent
       end
 
       def format_with_nested_fields(tag, time, record)
-        csv = (@cache[Thread.current] ||= CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts))
+        csv = csv_for_thread
         values = @accessors.map { |a| a.call(record) }
         line = (csv << values).string.dup
         # Need manual cleanup because CSV writer doesn't provide such method.

data/lib/fluent/plugin/in_http.rb

@@ -580,7 +580,7 @@ module Fluent::Plugin
         end
         path_info = uri.path
 
-        if (@add_query_params) 
+        if (@add_query_params)
 
           query_params = WEBrick::HTTPUtils.parse_query(uri.query)
 

data/lib/fluent/plugin/in_tail.rb

@@ -534,7 +534,7 @@ module Fluent::Plugin
       if @follow_inodes && new_inode.nil?
         # nil inode means the file disappeared, so we only need to stop it.
         @tails.delete(tail_watcher.path)
-        # https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632 
+        # https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632
         # Because of this problem, log duplication can occur during `rotate_wait`.
         # Need to set `rotate_wait 0` for a workaround.
         # Duplication will occur if `refresh_watcher` is called during the `rotate_wait`.

data/lib/fluent/supervisor.rb

@@ -440,7 +440,7 @@ module Fluent
         stop_immediately_at_unrecoverable_exit: true,
         root_dir: params['root_dir'],
         logger: $log,
-        log: $log.out,
+        log: $log&.out,
         log_level: params['log_level'],
         chuser: params['chuser'],
         chgroup: params['chgroup'],
@@ -742,7 +742,7 @@ module Fluent
       $log.enable_debug if system_config.log_level <= Fluent::Log::LEVEL_DEBUG
 
       $log.info "init #{process_type} logger",
-                path: actual_log_path, 
+                path: actual_log_path,
                 rotate_age: @log_rotate_age,
                 rotate_size: @log_rotate_size
     end
@@ -995,7 +995,7 @@ module Fluent
     def build_system_config(conf)
       system_config = SystemConfig.create(conf, @cl_opt[:strict_config_value])
       # Prefer the options explicitly specified in the command line
-      # 
+      #
       # TODO: There is a bug that `system_config.log.rotate_age/rotate_size` are
       # not merged with the command line options since they are not in
       # `SYSTEM_CONFIG_PARAMETERS`.

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.16.6'
+  VERSION = '1.16.8'
 
 end

data/lib/fluent/winsvc.rb

@@ -32,7 +32,7 @@ begin
   op.parse(ARGV)
   if opts[:service_name] == nil
     raise "Error: No Windows Service name set. Use '--service-name'"
-  end 
+  end
 
   def read_fluentdopt(service_name)
     Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\#{service_name}") do |reg|
@@ -48,25 +48,32 @@ begin
   end
 
   class FluentdService < Daemon
+    ERROR_WAIT_NO_CHILDREN = 128
+
     @pid = 0
     @service_name = ''
 
     def initialize(service_name)
       @service_name = service_name
     end
-    
+
     def service_main
       @pid = service_main_start(@service_name)
-      while running?
-        sleep 10
+      begin
+        loop do
+          sleep 5
+          break unless running?
+          raise Errno::ECHILD unless Process.waitpid2(@pid, Process::WNOHANG)
+        end
+      rescue Errno::ECHILD
+        @pid = 0
+        SetEvent(@@hStopEvent)
+        SetTheServiceStatus.call(SERVICE_STOPPED, ERROR_WAIT_NO_CHILDREN, 0, 0)
       end
     end
 
     def service_stop
-      set_event(@service_name)
-      if @pid > 0
-        Process.waitpid(@pid)
-      end
+      wait_supervisor_finished if @pid > 0
     end
 
     def service_paramchange
@@ -91,6 +98,29 @@ begin
       ev.set
       ev.close
     end
+
+    def repeat_set_event_several_times_until_success(event_name)
+      retries = 0
+      max_retries = 10
+      delay_sec = 3
+
+      begin
+        set_event(event_name)
+      rescue Errno::ENOENT
+        # This error occurs when the supervisor process has not yet created the event.
+        # If STOP is immediately executed, this state will occur.
+        # Retry `set_event' to wait for the initialization of the supervisor.
+        retries += 1
+        raise if max_retries < retries
+        sleep(delay_sec)
+        retry
+      end
+    end
+
+    def wait_supervisor_finished
+      repeat_set_event_several_times_until_success(@service_name)
+      Process.waitpid(@pid)
+    end
   end
 
   FluentdService.new(opts[:service_name]).mainloop

data/test/command/test_cat.rb

@@ -18,7 +18,7 @@ class TestFluentCat < ::Test::Unit::TestCase
     @primary = create_primary
     metadata = @primary.buffer.new_metadata
     @chunk = create_chunk(@primary, metadata, @es)
-    @port = unused_port
+    @port = unused_port(protocol: :all)
   end
 
   def teardown
@@ -87,7 +87,7 @@ class TestFluentCat < ::Test::Unit::TestCase
       d = create_driver
       d.run(expect_records: 1) do
         Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} --format msgpack secondary") do |stdin|
-          stdin.write(File.read(path))
+          stdin.write(File.read(path, File.size(path)))
           stdin.close
         end
       end

data/test/command/test_fluentd.rb

@@ -1175,7 +1175,7 @@ CONF
     end
   end
 
-  sub_test_case 'sahred socket options' do
+  sub_test_case 'shared socket options' do
     test 'enable shared socket by default' do
       conf = ""
       conf_path = create_conf_file('empty.conf', conf)

data/test/config/test_plugin_configuration.rb

@@ -19,9 +19,9 @@ module ConfigurationForPlugins
 
   class BooleanParamsWithoutValue < ::Test::Unit::TestCase
     CONFIG = <<CONFIG
-    flag1 
+    flag1
     flag2 # yaaaaaaaaaay
-    flag3 
+    flag3
     flag4 # yaaaaaaaaaay
     <child>
       flag1
@@ -37,10 +37,10 @@ module ConfigurationForPlugins
     </child>
     # with following whitespace
     <child>
-      flag1 
-      flag2 
-      flag3 
-      flag4 
+      flag1
+      flag2
+      flag3
+      flag4
     </child>
 CONFIG
 

data/test/helper.rb

@@ -71,17 +71,31 @@ end
 
 include Fluent::Test::Helpers
 
-def unused_port(num = 1, protocol: :tcp, bind: "0.0.0.0")
+def unused_port(num = 1, protocol:, bind: "0.0.0.0")
   case protocol
-  when :tcp
+  when :tcp, :tls
     unused_port_tcp(num)
   when :udp
     unused_port_udp(num, bind: bind)
+  when :all
+    unused_port_tcp_udp(num)
   else
     raise ArgumentError, "unknown protocol: #{protocol}"
   end
 end
 
+def unused_port_tcp_udp(num = 1)
+  raise "not support num > 1" if num > 1
+
+  # The default maximum number of file descriptors in macOS is 256.
+  # It might need to set num to a smaller value than that.
+  tcp_ports = unused_port_tcp(200)
+  port = unused_port_udp(1, port_list: tcp_ports)
+  raise "can't find unused port" unless port
+
+  port
+end
+
 def unused_port_tcp(num = 1)
   ports = []
   sockets = []
@@ -90,7 +104,7 @@ def unused_port_tcp(num = 1)
     sockets << s
     ports << s.addr[1]
   end
-  sockets.each{|s| s.close }
+  sockets.each(&:close)
   if num == 1
     return ports.first
   else
@@ -100,12 +114,15 @@ end
 
 PORT_RANGE_AVAILABLE = (1024...65535)
 
-def unused_port_udp(num = 1, bind: "0.0.0.0")
+def unused_port_udp(num = 1, port_list: [], bind: "0.0.0.0")
   family = IPAddr.new(IPSocket.getaddress(bind)).ipv4? ? ::Socket::AF_INET : ::Socket::AF_INET6
   ports = []
   sockets = []
-  while ports.size < num
-    port = rand(PORT_RANGE_AVAILABLE)
+
+  use_random_port = port_list.empty?
+  i = 0
+  loop do
+    port = use_random_port ? rand(PORT_RANGE_AVAILABLE) : port_list[i]
     u = UDPSocket.new(family)
     if (u.bind(bind, port) rescue nil)
       ports << port
@@ -113,8 +130,11 @@ def unused_port_udp(num = 1, bind: "0.0.0.0")
     else
       u.close
     end
+    i += 1
+    break if ports.size >= num
+    break if !use_random_port && i >= port_list.size
   end
-  sockets.each{|s| s.close }
+  sockets.each(&:close)
   if num == 1
     return ports.first
   else

data/test/plugin/out_forward/test_ack_handler.rb

@@ -111,11 +111,11 @@ class AckHandlerTest < Test::Unit::TestCase
     r, w = IO.pipe
     begin
       w.write(chunk_id)
-      stub(r).recv { |_|
+      def r.recv(arg)
         sleep(1) # To ensure that multiple threads select the socket before closing.
-        raise IOError, 'stream closed in another thread' if r.closed?
+        raise IOError, 'stream closed in another thread' if self.closed?
         MessagePack.pack({ 'ack' => Base64.encode64('chunk_id 111') })
-      }
+      end
       ack.enqueue(r)
 
       threads = []

data/test/plugin/out_forward/test_socket_cache.rb

@@ -17,7 +17,7 @@ class SocketCacheTest < Test::Unit::TestCase
       assert_equal(socket, c.checkout_or('key') { socket })
       c.checkin(socket)
 
-      sock = dont_allow(mock!).open
+      sock = mock!.open.never.subject
       assert_equal(socket, c.checkout_or('key') { sock.open })
     end
 
@@ -130,7 +130,7 @@ class SocketCacheTest < Test::Unit::TestCase
 
       c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
       sock = mock!.close { 'closed' }.subject
-      sock2 = dont_allow(mock!).close
+      sock2 = mock!.close.never.subject
       stub(sock).inspect
       stub(sock2).inspect
 
@@ -154,7 +154,7 @@ class SocketCacheTest < Test::Unit::TestCase
       Timecop.freeze(Time.parse('2016-04-13 14:00:00 +0900'))
 
       c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
-      sock = dont_allow(mock!).close
+      sock = mock!.close.never.subject
       stub(sock).inspect
       c.checkout_or('key') { sock }
 

data/test/plugin/test_buffer.rb

@@ -946,9 +946,9 @@ class BufferTest < Test::Unit::TestCase
       # 1. `write_once`: 42 [events] * 1 [stream]
       # 2. `write_step_by_step`: 4 [events]* 10 [streams] + 2 [events] * 1 [stream]
       # 3. `write_step_by_step` (by `ShouldRetry`): 1 [event] * 42 [streams]
-      # 
+      #
       # Example of staged chunk lock behavior:
-      # 
+      #
       # 1. mon_enter in write_step_by_step
       # 2. ShouldRetry occurs
       # 3. mon_exit in write_step_by_step

data/test/plugin/test_filter_grep.rb

@@ -328,7 +328,7 @@ class GrepFilterTest < Test::Unit::TestCase
       end
 
       test "don't raise an exception" do
-        assert_nothing_raised { 
+        assert_nothing_raised {
           filter(%[regexp1 message WARN], ["\xff".force_encoding('UTF-8')])
         }
       end

data/test/plugin/test_in_forward.rb

@@ -18,7 +18,8 @@ class ForwardInputTest < Test::Unit::TestCase
     Fluent::Test.setup
     @responses = []  # for testing responses after sending data
     @d = nil
-    @port = unused_port
+    # forward plugin uses TCP and UDP sockets on the same port number
+    @port = unused_port(protocol: :all)
   end
 
   def teardown

data/test/plugin/test_in_http.rb

@@ -18,7 +18,7 @@ class HttpInputTest < Test::Unit::TestCase
 
   def setup
     Fluent::Test.setup
-    @port = unused_port
+    @port = unused_port(protocol: :tcp)
   end
 
   def teardown
@@ -982,7 +982,7 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal ["403", "403"], res_codes
   end
 
-  def test_add_query_params 
+  def test_add_query_params
     d = create_driver(config + "add_query_params true")
     assert_equal true, d.instance.add_query_params
 

data/test/plugin/test_in_monitor_agent.rb

@@ -392,7 +392,7 @@ EOC
     end
 
     test "emit" do
-      port = unused_port
+      port = unused_port(protocol: :tcp)
       d = create_driver("
   @type monitor_agent
   bind '127.0.0.1'
@@ -451,7 +451,7 @@ EOC
 
   sub_test_case "servlets" do
     setup do
-      @port = unused_port
+      @port = unused_port(protocol: :tcp)
       # check @type and type in one configuration
       conf = <<-EOC
 <source>
@@ -759,7 +759,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
     end
 
     setup do
-      @port = unused_port
+      @port = unused_port(protocol: :tcp)
       # check @type and type in one configuration
       conf = <<-EOC
 <source>
@@ -840,7 +840,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
 
   sub_test_case "check the port number of http server" do
     test "on single worker environment" do
-      port = unused_port
+      port = unused_port(protocol: :tcp)
       d = create_driver("
   @type monitor_agent
   bind '127.0.0.1'
@@ -851,7 +851,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
     end
 
     test "worker_id = 2 on multi worker environment" do
-      port = unused_port
+      port = unused_port(protocol: :tcp)
       Fluent::SystemConfig.overwrite_system_config('workers' => 4) do
         d = Fluent::Test::Driver::Input.new(Fluent::Plugin::MonitorAgentInput)
         d.instance.instance_eval{ @_fluentd_worker_id = 2 }
@@ -905,7 +905,7 @@ EOC
     end
 
     test "plugins have a variable named buffer does not throws NoMethodError" do
-      port = unused_port
+      port = unused_port(protocol: :tcp)
       d = create_driver("
   @type monitor_agent
   bind '127.0.0.1'