fluentd 1.16.7-x86-mingw32 → 1.16.8-x86-mingw32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fcc3a30a9e314d87517f7be5560f5ff382d3335b0320be04ac6b3bb489bbdd8c
-  data.tar.gz: e8a6e2ff32150bf564036724aa5e39ee45c933ea391498163a70b4aa3242473d
+  metadata.gz: 0c5ae232fdfccc14424b4d022e8d3e4ca9bd05b3b7c8a8de10508263c6ea651c
+  data.tar.gz: 60d0abe234b4493236956d88f63d5807c7124682dec6333d3966148f66efa1dc
 SHA512:
-  metadata.gz: d13313bad71d1bf23044f1a8fef3157c8e5392024cf6d2f5015fd4a2ab57eab0fc86b52fa9191705a82536cfafadafb7c19b975bc6c7d68f925ac132590a7278
-  data.tar.gz: e0300f26f6713d5cada654b7774b0f0a25a4593cc2f961e484c54c5e15effdbdc87b17e2398f3f4430ca4e92e0a92e014223534e54cf44772a2c55918c73fdc6
+  metadata.gz: e606b2d087be49f0ddaa3dff9d4dd0b62ad8e70e0a3d5f044df875b53606dc46ab0755d33edca16bd10b9c386be4e87ea10ef47a34a4cecda4f635e32f2236da
+  data.tar.gz: a19cde7e7a898b1c93f404dfdc94b9828140f4793c118a3c596e49c7ba77e317c33ac958cb9cc1d4fe33b6ea7cadd2962ed6d4373e6f865e417fa44d742a48f9

data/.github/workflows/test.yml

@@ -12,6 +12,10 @@ on:
       - '*.md'
       - 'lib/fluent/version.rb'
 
+concurrency:
+  group: ${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ${{ matrix.os }}

data/CHANGELOG.md

@@ -1,5 +1,33 @@
 # v1.16
 
+## Release v1.16.8 - 2025/05/01
+
+### Bug Fix
+
+* winsvc: Stop the service when the supervisor is dead
+  https://github.com/fluent/fluentd/pull/4942
+* formatter_csv: Fix memory leak
+  https://github.com/fluent/fluentd/pull/4920
+
+### Misc
+
+* Add fiddle as dependency gem for Ruby 3.5 on Windows
+  https://github.com/fluent/fluentd/pull/4919
+* Refactoring code
+  https://github.com/fluent/fluentd/pull/4921
+  https://github.com/fluent/fluentd/pull/4922
+  https://github.com/fluent/fluentd/pull/4926
+  https://github.com/fluent/fluentd/pull/4943
+* CI improvemnts
+  https://github.com/fluent/fluentd/pull/4821
+  https://github.com/fluent/fluentd/pull/4850
+  https://github.com/fluent/fluentd/pull/4851
+  https://github.com/fluent/fluentd/pull/4862
+  https://github.com/fluent/fluentd/pull/4915
+  https://github.com/fluent/fluentd/pull/4923
+  https://github.com/fluent/fluentd/pull/4925
+  https://github.com/fluent/fluentd/pull/4927
+
 ## Release v1.16.7 - 2025/01/29
 
 ### Bug Fix

data/fluentd.gemspec

@@ -32,9 +32,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("console", ["< 1.24"])
 
   # gems that aren't default gems as of Ruby 3.5
-  # logger 1.6.3 or later cause bug on windows,
-  # hold on 1.6.2 for a while. see https://github.com/ruby/logger/issues/107
-  gem.add_runtime_dependency("logger", ["1.6.2"])
+  gem.add_runtime_dependency("logger", ["~> 1.6"])
 
   # build gem for a certain platform. see also Rakefile
   fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
@@ -44,6 +42,9 @@ Gem::Specification.new do |gem|
     gem.add_runtime_dependency("win32-ipc", ["~> 0.7.0"])
     gem.add_runtime_dependency("win32-event", ["~> 0.6.3"])
     gem.add_runtime_dependency("certstore_c", ["~> 0.1.7"])
+
+    # gems that aren't default gems as of Ruby 3.5
+    gem.add_runtime_dependency("fiddle", ["~> 1.1"])
   end
 
   gem.add_development_dependency("rake", ["~> 13.0"])

data/lib/fluent/compat/formatter.rb

@@ -101,6 +101,12 @@ module Fluent
 
       class CsvFormatter < Fluent::Plugin::CsvFormatter
         # TODO: warn when deprecated
+
+        # Do not cache because it is hard to consider the thread key correctly.
+        # (We can try, but it would be low priority.)
+        def csv_cacheable?
+          false
+        end
       end
 
       class SingleValueFormatter < Fluent::Plugin::SingleValueFormatter

data/lib/fluent/config/parser.rb

@@ -92,10 +92,23 @@ module Fluent
         else
           basepath = '/'
           fname = path
-          require 'open-uri'
-          URI.open(uri) {|f|
+          parser_proc = ->(f) {
             Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
           }
+
+          case u.scheme
+          when 'http', 'https', 'ftp'
+            # URI#open can be able to handle URIs for http, https and ftp.
+            require 'open-uri'
+            u.open(&parser_proc)
+          else
+            # TODO: This case should be handled in the previous if condition. Glob is not applied to some Windows path formats.
+            # 'c:/path/to/file' will be passed as URI, 'uri' and 'u.path' will be:
+            #   - uri is 'c:/path/to/file'
+            #   - u.path is '/path/to/file' and u.scheme is 'c'
+            # Therefore, the condition of the if statement above is not met and it is handled here.
+            File.open(uri, &parser_proc)
+          end
         end
 
       rescue SystemCallError => e
@@ -104,4 +117,3 @@ module Fluent
     end
   end
 end
-

data/lib/fluent/config/v1_parser.rb

@@ -172,7 +172,7 @@ module Fluent
           require 'open-uri'
           basepath = '/'
           fname = path
-          data = URI.open(uri) { |f| f.read }
+          data = u.open { |f| f.read }
           data.force_encoding('UTF-8')
           ss = StringScanner.new(data)
           V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)

data/lib/fluent/plugin/formatter_csv.rb

@@ -35,6 +35,22 @@ module Fluent
       config_param :fields, :array, value_type: :string
       config_param :add_newline, :bool, default: true
 
+      def csv_cacheable?
+        !!owner
+      end
+
+      def csv_thread_key
+        csv_cacheable? ? "#{owner.plugin_id}_csv_formatter_#{@usage}_csv" : nil
+      end
+
+      def csv_for_thread
+        if csv_cacheable?
+          Thread.current[csv_thread_key] ||= CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts)
+        else
+          CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts)
+        end
+      end
+
       def configure(conf)
         super
 
@@ -51,12 +67,10 @@ module Fluent
 
         @generate_opts = {col_sep: @delimiter, force_quotes: @force_quotes, headers: @fields,
                           row_sep: @add_newline ? :auto : "".force_encoding(Encoding::ASCII_8BIT)}
-        # Cache CSV object per thread to avoid internal state sharing
-        @cache = {}
       end
 
       def format(tag, time, record)
-        csv = (@cache[Thread.current] ||= CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts))
+        csv = csv_for_thread
         line = (csv << record).string.dup
         # Need manual cleanup because CSV writer doesn't provide such method.
         csv.rewind
@@ -65,7 +79,7 @@ module Fluent
       end
 
       def format_with_nested_fields(tag, time, record)
-        csv = (@cache[Thread.current] ||= CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts))
+        csv = csv_for_thread
         values = @accessors.map { |a| a.call(record) }
         line = (csv << values).string.dup
         # Need manual cleanup because CSV writer doesn't provide such method.

data/lib/fluent/plugin/in_http.rb

@@ -580,7 +580,7 @@ module Fluent::Plugin
         end
         path_info = uri.path
 
-        if (@add_query_params) 
+        if (@add_query_params)
 
           query_params = WEBrick::HTTPUtils.parse_query(uri.query)
 

data/lib/fluent/plugin/in_tail.rb

@@ -534,7 +534,7 @@ module Fluent::Plugin
       if @follow_inodes && new_inode.nil?
         # nil inode means the file disappeared, so we only need to stop it.
         @tails.delete(tail_watcher.path)
-        # https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632 
+        # https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632
         # Because of this problem, log duplication can occur during `rotate_wait`.
         # Need to set `rotate_wait 0` for a workaround.
         # Duplication will occur if `refresh_watcher` is called during the `rotate_wait`.

data/lib/fluent/supervisor.rb

@@ -742,7 +742,7 @@ module Fluent
       $log.enable_debug if system_config.log_level <= Fluent::Log::LEVEL_DEBUG
 
       $log.info "init #{process_type} logger",
-                path: actual_log_path, 
+                path: actual_log_path,
                 rotate_age: @log_rotate_age,
                 rotate_size: @log_rotate_size
     end
@@ -995,7 +995,7 @@ module Fluent
     def build_system_config(conf)
       system_config = SystemConfig.create(conf, @cl_opt[:strict_config_value])
       # Prefer the options explicitly specified in the command line
-      # 
+      #
       # TODO: There is a bug that `system_config.log.rotate_age/rotate_size` are
       # not merged with the command line options since they are not in
       # `SYSTEM_CONFIG_PARAMETERS`.

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.16.7'
+  VERSION = '1.16.8'
 
 end

data/lib/fluent/winsvc.rb

@@ -32,7 +32,7 @@ begin
   op.parse(ARGV)
   if opts[:service_name] == nil
     raise "Error: No Windows Service name set. Use '--service-name'"
-  end 
+  end
 
   def read_fluentdopt(service_name)
     Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\#{service_name}") do |reg|
@@ -48,27 +48,32 @@ begin
   end
 
   class FluentdService < Daemon
+    ERROR_WAIT_NO_CHILDREN = 128
+
     @pid = 0
     @service_name = ''
 
     def initialize(service_name)
       @service_name = service_name
     end
-    
+
     def service_main
       @pid = service_main_start(@service_name)
-      while running?
-        sleep 10
+      begin
+        loop do
+          sleep 5
+          break unless running?
+          raise Errno::ECHILD unless Process.waitpid2(@pid, Process::WNOHANG)
+        end
+      rescue Errno::ECHILD
+        @pid = 0
+        SetEvent(@@hStopEvent)
+        SetTheServiceStatus.call(SERVICE_STOPPED, ERROR_WAIT_NO_CHILDREN, 0, 0)
       end
     end
 
     def service_stop
-      if @pid <= 0
-        set_event(@service_name)
-        return
-      end
-
-      wait_supervisor_finished
+      wait_supervisor_finished if @pid > 0
     end
 
     def service_paramchange

data/test/config/test_plugin_configuration.rb

@@ -19,9 +19,9 @@ module ConfigurationForPlugins
 
   class BooleanParamsWithoutValue < ::Test::Unit::TestCase
     CONFIG = <<CONFIG
-    flag1 
+    flag1
     flag2 # yaaaaaaaaaay
-    flag3 
+    flag3
     flag4 # yaaaaaaaaaay
     <child>
       flag1
@@ -37,10 +37,10 @@ module ConfigurationForPlugins
     </child>
     # with following whitespace
     <child>
-      flag1 
-      flag2 
-      flag3 
-      flag4 
+      flag1
+      flag2
+      flag3
+      flag4
     </child>
 CONFIG
 

data/test/plugin/test_buffer.rb

@@ -946,9 +946,9 @@ class BufferTest < Test::Unit::TestCase
       # 1. `write_once`: 42 [events] * 1 [stream]
       # 2. `write_step_by_step`: 4 [events]* 10 [streams] + 2 [events] * 1 [stream]
       # 3. `write_step_by_step` (by `ShouldRetry`): 1 [event] * 42 [streams]
-      # 
+      #
       # Example of staged chunk lock behavior:
-      # 
+      #
       # 1. mon_enter in write_step_by_step
       # 2. ShouldRetry occurs
       # 3. mon_exit in write_step_by_step

data/test/plugin/test_filter_grep.rb

@@ -328,7 +328,7 @@ class GrepFilterTest < Test::Unit::TestCase
       end
 
       test "don't raise an exception" do
-        assert_nothing_raised { 
+        assert_nothing_raised {
           filter(%[regexp1 message WARN], ["\xff".force_encoding('UTF-8')])
         }
       end

data/test/plugin/test_in_http.rb

@@ -982,7 +982,7 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal ["403", "403"], res_codes
   end
 
-  def test_add_query_params 
+  def test_add_query_params
     d = create_driver(config + "add_query_params true")
     assert_equal true, d.instance.add_query_params
 

data/test/plugin/test_in_tail.rb

@@ -1611,6 +1611,7 @@ class TailInputTest < Test::Unit::TestCase
 
       cleanup_file("#{@tmp_dir}/tail.txt")
       waiting(20) { sleep 0.1 until Dir.glob("#{@tmp_dir}/*.txt").size == 0 } # Ensure file is deleted on Windows
+      waiting(5) { sleep 0.1 until d.logs.last.include?("detected rotation") }
       waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size <= 0 }
 
       assert_equal(

data/test/plugin/test_output.rb

@@ -1111,7 +1111,7 @@ class OutputTest < Test::Unit::TestCase
         config: config_element(
           "ROOT", "", {},
           [
-            config_element("buffer", "", {}), 
+            config_element("buffer", "", {}),
             config_element("secondary", "", {"@type" => "test", "name" => "test"}),
           ]
         ),

data/test/plugin/test_output_as_buffered.rb

@@ -1343,6 +1343,7 @@ class BufferedOutputTest < Test::Unit::TestCase
       hash = {
         'flush_interval' => 10,
         'flush_thread_count' => 1,
+        'flush_thread_interval' => 0.1,
         'flush_thread_burst_interval' => 0.1,
         'chunk_limit_size' => 1024,
       }
@@ -1445,7 +1446,7 @@ class BufferedOutputTest < Test::Unit::TestCase
 
       assert{ @i.buffer.stage.size == 3 }
 
-      # to trigger try_flush with flush_thread_burst_interval
+      # to trigger try_flush with flush_thread_interval
       Timecop.freeze( Time.parse('2016-04-13 14:04:11 +0900') )
       @i.enqueue_thread_wait
       Timecop.freeze( Time.parse('2016-04-13 14:04:12 +0900') )
@@ -1454,7 +1455,6 @@ class BufferedOutputTest < Test::Unit::TestCase
       @i.enqueue_thread_wait
       Timecop.freeze( Time.parse('2016-04-13 14:04:14 +0900') )
       @i.enqueue_thread_wait
-      @i.flush_thread_wakeup
 
       assert{ @i.buffer.stage.size == 0 }
 

data/test/plugin/test_output_as_buffered_retries.rb

@@ -941,7 +941,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.enqueue_thread_wait
 
       @i.flush_thread_wakeup
-      waiting(4){ Thread.pass until @i.write_count > 0 }
+      waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 }
       waiting(4) do
         state = @i.instance_variable_get(:@output_flush_threads).first
         state.thread.status == 'sleep'
@@ -953,7 +953,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       now = @i.next_flush_time
       Timecop.freeze( now )
       @i.flush_thread_wakeup
-      waiting(4){ Thread.pass until @i.write_count > 1 }
+      waiting(4){ Thread.pass until @i.write_count > 1 && @i.num_errors > 1 }
       waiting(4) do
         state = @i.instance_variable_get(:@output_flush_threads).first
         state.thread.status == 'sleep'

data/test/plugin/test_parser_csv.rb

@@ -164,7 +164,7 @@ class CSVParserTest < ::Test::Unit::TestCase
       text = 'a"b,"a"""c"'
       assert_raise(CSV::MalformedCSVError) {
         normal.instance.parse(text) { |t, r| }
-      } 
+      }
       assert_nothing_raised {
         # generate broken record
         fast.instance.parse(text) { |t, r| }

data/test/plugin_helper/test_server.rb

@@ -668,12 +668,12 @@ class ServerPluginHelperTest < Test::Unit::TestCase
 
       received = ""
       responses = []
-      port = unused_port(protocol: :udp)
+      port = unused_port(protocol: :udp, bind: "::1")
       @d.server_create_udp(:s, port, bind: "::1", max_bytes: 128) do |data, sock|
         received << data
         sock.write "ack\n"
       end
-      bind_port = unused_port(protocol: :udp)
+      bind_port = unused_port(protocol: :udp, bind: "::1")
       3.times do
         begin
           sock = UDPSocket.new(Socket::AF_INET6)

data/test/test_supervisor.rb

@@ -214,6 +214,9 @@ class SupervisorTest < ::Test::Unit::TestCase
   def test_cont_in_main_process_signal_handlers
     omit "Windows cannot handle signals" if Fluent.windows?
 
+    # https://github.com/fluent/fluentd/issues/4063
+    GC.start
+
     sv = Fluent::Supervisor.new({})
     sv.send(:install_main_process_signal_handlers)
 
@@ -229,6 +232,9 @@ class SupervisorTest < ::Test::Unit::TestCase
   def test_term_cont_in_main_process_signal_handlers
     omit "Windows cannot handle signals" if Fluent.windows?
 
+    # https://github.com/fluent/fluentd/issues/4063
+    GC.start
+
     create_debug_dummy_logger
 
     sv = Fluent::Supervisor.new({})
@@ -296,6 +302,9 @@ class SupervisorTest < ::Test::Unit::TestCase
   def test_cont_in_supervisor_signal_handler
     omit "Windows cannot handle signals" if Fluent.windows?
 
+    # https://github.com/fluent/fluentd/issues/4063
+    GC.start
+
     server = DummyServer.new
     server.install_supervisor_signal_handlers
 
@@ -311,6 +320,9 @@ class SupervisorTest < ::Test::Unit::TestCase
   def test_term_cont_in_supervisor_signal_handler
     omit "Windows cannot handle signals" if Fluent.windows?
 
+    # https://github.com/fluent/fluentd/issues/4063
+    GC.start
+
     server = DummyServer.new
     server.install_supervisor_signal_handlers
 
@@ -390,34 +402,31 @@ class SupervisorTest < ::Test::Unit::TestCase
   def test_supervisor_event_dump_windows
     omit "Only for Windows, alternative to UNIX signals" unless Fluent.windows?
 
+    # https://github.com/fluent/fluentd/issues/4063
+    GC.start
+
+    ENV['SIGDUMP_PATH'] = TMP_DIR + "/sigdump.log"
+
     server = DummyServer.new
     def server.config
       {:signame => "TestFluentdEvent"}
     end
     server.install_windows_event_handler
 
-    assert_rr do
-      # Have to use mock because `Sigdump.dump` seems to be somehow incompatible with RR.
-      # The `mock(server).restart(true) { nil }` line in `test_rpc_server_windows` cause the next error.
-      # Failure: test_supervisor_event_dump_windows(SupervisorTest):
-      #   class()
-      #   Called 0 times.
-      #   Expected 1 times.
-      # .../Ruby26-x64/lib/ruby/gems/2.6.0/gems/sigdump-0.2.4/lib/sigdump.rb:74:in `block in dump_object_count'
-      #     73: ObjectSpace.each_object {|o|
-      #     74:   c = o.class <-- HERE!
-      mock(Sigdump).dump(anything)
-
-      begin
-        sleep 0.1 # Wait for starting windows event thread
-        event = Win32::Event.open("TestFluentdEvent_CONT")
-        event.set
-        event.close
-        sleep 1.0 # Wait for dumping
-      ensure
-        server.stop_windows_event_thread
-      end
+    begin
+      sleep 0.1 # Wait for starting windows event thread
+      event = Win32::Event.open("TestFluentdEvent_CONT")
+      event.set
+      event.close
+      sleep 1.0 # Wait for dumping
+    ensure
+      server.stop_windows_event_thread
     end
+
+    result_filepaths = Dir.glob("#{TMP_DIR}/*")
+    assert {result_filepaths.length > 0}
+  ensure
+    ENV.delete('SIGDUMP_PATH')
   end
 
   data(:ipv4 => ["0.0.0.0", "127.0.0.1", false],

data/test/test_tls.rb

@@ -9,7 +9,7 @@ class UniqueIdTest < Test::Unit::TestCase
   TEST_TLS1_2_CASES = {
     'New TLS v1.2' => :'TLS1_2',
     'Old TLS v1.2' => :'TLSv1_2'
-  } 
+  }
   TEST_TLS_CASES = TEST_TLS1_1_CASES.merge(TEST_TLS1_2_CASES)
 
   sub_test_case 'constants' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.16.7
+  version: 1.16.8
 platform: x86-mingw32
 authors:
 - Sadayuki Furuhashi
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2025-01-29 00:00:00.000000000 Z
+date: 2025-05-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -218,16 +218,16 @@ dependencies:
   name: logger
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.2
+        version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.2
+        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: win32-service
   requirement: !ruby/object:Gem::Requirement
@@ -284,6 +284,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.7
+- !ruby/object:Gem::Dependency
+  name: fiddle
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -1025,7 +1039,7 @@ homepage: https://www.fluentd.org/
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -1041,7 +1055,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.4.19
-signing_key:
+signing_key: 
 specification_version: 4
 summary: Fluentd event collector
 test_files: