fluentd 1.17.1 → 1.19.0

data/lib/fluent/system_config.rb

@@ -16,6 +16,7 @@
 
 require 'fluent/configurable'
 require 'fluent/config/element'
+require 'fluent/env'
 
 module Fluent
   class SystemConfig
@@ -25,10 +26,11 @@ module Fluent
       :workers, :restart_worker_interval, :root_dir, :log_level,
       :suppress_repeated_stacktrace, :emit_error_log_interval, :suppress_config_dump,
       :log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,
-      :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
+      :without_source, :with_source_only, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
       :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket,
-      :metrics, :enable_input_metrics, :enable_size_metrics, :enable_jit
+      :metrics, :enable_input_metrics, :enable_size_metrics, :enable_jit, :source_only_buffer,
+      :config_include_dir
     ]
 
     config_param :workers,   :integer, default: 1
@@ -41,14 +43,15 @@ module Fluent
     config_param :emit_error_log_interval,      :time, default: nil
     config_param :suppress_config_dump, :bool, default: nil
     config_param :log_event_verbose,    :bool, default: nil
-    config_param :without_source,  :bool, default: nil
+    config_param :without_source, :bool, default: nil
+    config_param :with_source_only, :bool, default: nil
     config_param :rpc_endpoint,    :string, default: nil
     config_param :enable_get_dump, :bool, default: nil
     config_param :process_name,    :string, default: nil
     config_param :strict_config_value, :bool, default: nil
     config_param :enable_msgpack_time_support, :bool, default: nil
     config_param :disable_shared_socket, :bool, default: nil
-    config_param :enable_input_metrics, :bool, default: nil
+    config_param :enable_input_metrics, :bool, default: true
     config_param :enable_size_metrics, :bool, default: nil
     config_param :enable_jit, :bool, default: false
     config_param :file_permission, default: nil do |v|
@@ -57,7 +60,9 @@ module Fluent
     config_param :dir_permission, default: nil do |v|
       v.to_i(8)
     end
+    config_param :config_include_dir, default: Fluent::DEFAULT_CONFIG_INCLUDE_DIR
     config_section :log, required: false, init: true, multi: false do
+      config_param :path, :string, default: nil
       config_param :format, :enum, list: [:text, :json], default: :text
       config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S %z'
       config_param :rotate_age, default: nil do |v|
@@ -74,6 +79,7 @@ module Fluent
         end
       end
       config_param :rotate_size, :size, default: nil
+      config_param :forced_stacktrace_level, :enum, list: [:none, :trace, :debug, :info, :warn, :error, :fatal], default: :none
     end
 
     config_section :counter_server, multi: false do
@@ -103,6 +109,20 @@ module Fluent
       config_param :labels, :hash, default: {}
     end
 
+    config_section :source_only_buffer, init: true, multi: false do
+      config_param :flush_thread_count, :integer, default: 1
+      config_param :overflow_action, :enum, list: [:throw_exception, :block, :drop_oldest_chunk], default: :drop_oldest_chunk
+      config_param :path, :string, default: nil
+      config_param :flush_interval, :time, default: nil
+      config_param :chunk_limit_size, :size, default: nil
+      config_param :total_limit_size, :size, default: nil
+      config_param :compress, :enum, list: [:text, :gzip], default: nil
+    end
+
+    def force_stacktrace_level?
+      @log.forced_stacktrace_level != :none
+    end
+
     def self.create(conf, strict_config_value=false)
       systems = conf.elements(name: 'system')
       return SystemConfig.new if systems.empty?
@@ -147,6 +167,7 @@ module Fluent
       end
 
       @log_level = Log.str_to_level(@log_level.to_s) if @log_level
+      @log[:forced_stacktrace_level] = Log.str_to_level(@log.forced_stacktrace_level.to_s) if force_stacktrace_level?
     end
 
     def dup
@@ -171,13 +192,13 @@ module Fluent
         unless defined?($_system_config)
           $_system_config = nil
         end
-        (instance_variable_defined?("@_system_config") && @_system_config) ||
+        (instance_variable_defined?(:@_system_config) && @_system_config) ||
           $_system_config || Fluent::Engine.system_config
       end
 
       def system_config_override(opts={})
         require 'fluent/engine'
-        if !instance_variable_defined?("@_system_config") || @_system_config.nil?
+        if !instance_variable_defined?(:@_system_config) || @_system_config.nil?
           @_system_config = (defined?($_system_config) && $_system_config ? $_system_config : Fluent::Engine.system_config).dup
         end
         opts.each_pair do |key, value|

data/lib/fluent/test/base.rb

@@ -32,7 +32,7 @@ module Fluent
             #   klass.dup is worse because its ancestors does NOT include original class name
             klass_name = klass.name
             klass = Class.new(klass)
-            klass.define_singleton_method("name") { klass_name }
+            klass.define_singleton_method(:name) { klass_name }
             klass.module_eval(&block)
           end
           @instance = klass.new

data/lib/fluent/test/driver/base.rb

@@ -185,7 +185,7 @@ module Fluent
           end
         end
 
-        def run_actual(timeout: DEFAULT_TIMEOUT, &block)
+        def run_actual(timeout: DEFAULT_TIMEOUT)
           if @instance.respond_to?(:_threads)
             sleep 0.1 until @instance._threads.values.all?(&:alive?)
           end
@@ -201,7 +201,7 @@ module Fluent
           return_value = nil
           begin
             Timeout.timeout(timeout * 2) do |sec|
-              return_value = block.call if block_given?
+              return_value = yield if block_given?
             end
           rescue Timeout::Error
             raise TestTimedOut, "Test case timed out with hard limit."

data/lib/fluent/test/filter_test.rb

@@ -59,9 +59,9 @@ module Fluent
       alias_method :emits, :filtered_as_array # emits is for consistent with other drivers
 
       # Almost filters don't use threads so default is 0. It reduces test time.
-      def run(num_waits = 0, &block)
+      def run(num_waits = 0)
         super(num_waits) {
-          block.call if block
+          yield if block_given?
 
           @events.each { |tag, es|
             processed = @instance.filter_stream(tag, es)

data/lib/fluent/test/formatter_test.rb

@@ -28,7 +28,7 @@ module Fluent
             #   klass.dup is worse because its ancestors does NOT include original class name
             klass_name = klass_or_str.name
             klass_or_str = Class.new(klass_or_str)
-            klass_or_str.define_singleton_method("name") { klass_name }
+            klass_or_str.define_singleton_method(:name) { klass_name }
             klass_or_str.module_eval(&block)
           end
           @instance = klass_or_str.new

data/lib/fluent/test/helpers.rb

@@ -50,6 +50,10 @@ EOT
         end
       end
 
+      def event_time_without_nsec(str=nil, format: nil)
+        Fluent::EventTime.new(event_time(str, format: format))
+      end
+
       def with_timezone(tz)
         oldtz, ENV['TZ'] = ENV['TZ'], tz
         yield

data/lib/fluent/test/input_test.rb

@@ -109,7 +109,7 @@ module Fluent
         end
       end
 
-      def run(num_waits = 10, &block)
+      def run(num_waits = 10)
         m = method(:emit_stream)
         unless Engine.singleton_class.ancestors.include?(EmitStreamWrapper)
           Engine.singleton_class.prepend EmitStreamWrapper
@@ -121,7 +121,7 @@ module Fluent
         instance.router.emit_stream_callee = m
 
         super(num_waits) {
-          block.call if block
+          yield if block_given?
 
           if @expected_emits_length || @expects || @run_post_conditions
             # counters for emits and emit_streams

data/lib/fluent/test/output_test.rb

@@ -71,10 +71,10 @@ module Fluent
         (@expected_buffer ||= '') << str
       end
 
-      def run(num_waits = 10, &block)
+      def run(num_waits = 10)
         result = nil
         super(num_waits) {
-          block.call if block
+          yield if block_given?
 
           es = ArrayEventStream.new(@entries)
           buffer = @instance.format_stream(@tag, es)
@@ -119,10 +119,10 @@ module Fluent
         (@expected_buffer ||= '') << str
       end
 
-      def run(&block)
+      def run
         result = []
         super {
-          block.call if block
+          yield if block_given?
 
           buffer = ''
           lines = {}

data/lib/fluent/test/parser_test.rb

@@ -27,7 +27,7 @@ module Fluent
             #   klass.dup is worse because its ancestors does NOT include original class name
             klass_name = klass_or_str.name
             klass_or_str = Class.new(klass_or_str)
-            klass_or_str.define_singleton_method("name") { klass_name }
+            klass_or_str.define_singleton_method(:name) { klass_name }
             klass_or_str.module_eval(&block)
           end
           case klass_or_str.instance_method(:initialize).arity

data/lib/fluent/tls.rb

@@ -76,6 +76,30 @@ module Fluent
       ctx
     end
     module_function :set_version_to_context
+
+    def set_version_to_options(opt, version, min_version, max_version)
+      if MIN_MAX_AVAILABLE
+        case
+        when min_version.nil? && max_version.nil?
+          min_version = METHODS_MAP[version] || version
+          max_version = METHODS_MAP[version] || version
+        when min_version.nil? && max_version
+          raise Fluent::ConfigError, "When you set max_version, must set min_version together"
+        when min_version && max_version.nil?
+          raise Fluent::ConfigError, "When you set min_version, must set max_version together"
+        else
+          min_version = METHODS_MAP[min_version] || min_version
+          max_version = METHODS_MAP[max_version] || max_version
+        end
+        opt[:min_version] = min_version
+        opt[:max_version] = max_version
+      else
+        opt[:ssl_version] = METHODS_MAP[version] || version
+      end
+
+      opt
+    end
+    module_function :set_version_to_options
   end
 end
 

data/lib/fluent/variable_store.rb

@@ -15,7 +15,7 @@
 #
 
 module Fluent
-  # VariableStore provides all pluigns with the way to shared variable without using class variable
+  # VariableStore provides all plugins with the way to shared variable without using class variable
   # it's for safe reloading mechanism
   class VariableStore
     @data = {}

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.17.1'
+  VERSION = '1.19.0'
 
 end

data/lib/fluent/winsvc.rb

@@ -32,7 +32,7 @@ begin
   op.parse(ARGV)
   if opts[:service_name] == nil
     raise "Error: No Windows Service name set. Use '--service-name'"
-  end 
+  end
 
   def read_fluentdopt(service_name)
     Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\#{service_name}") do |reg|
@@ -48,25 +48,32 @@ begin
   end
 
   class FluentdService < Daemon
+    ERROR_WAIT_NO_CHILDREN = 128
+
     @pid = 0
     @service_name = ''
 
     def initialize(service_name)
       @service_name = service_name
     end
-    
+
     def service_main
       @pid = service_main_start(@service_name)
-      while running?
-        sleep 10
+      begin
+        loop do
+          sleep 5
+          break unless running?
+          raise Errno::ECHILD if Process.waitpid(@pid, Process::WNOHANG)
+        end
+      rescue Errno::ECHILD
+        @pid = 0
+        SetEvent(@@hStopEvent)
+        SetTheServiceStatus.call(SERVICE_STOPPED, ERROR_WAIT_NO_CHILDREN, 0, 0)
       end
     end
 
     def service_stop
-      set_event(@service_name)
-      if @pid > 0
-        Process.waitpid(@pid)
-      end
+      wait_supervisor_finished if @pid > 0
     end
 
     def service_paramchange
@@ -91,6 +98,29 @@ begin
       ev.set
       ev.close
     end
+
+    def repeat_set_event_several_times_until_success(event_name)
+      retries = 0
+      max_retries = 10
+      delay_sec = 3
+
+      begin
+        set_event(event_name)
+      rescue Errno::ENOENT
+        # This error occurs when the supervisor process has not yet created the event.
+        # If STOP is immediately executed, this state will occur.
+        # Retry `set_event' to wait for the initialization of the supervisor.
+        retries += 1
+        raise if max_retries < retries
+        sleep(delay_sec)
+        retry
+      end
+    end
+
+    def wait_supervisor_finished
+      repeat_set_event_several_times_until_success(@service_name)
+      Process.waitpid(@pid)
+    end
   end
 
   FluentdService.new(opts[:service_name]).mainloop

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.17.1
+  version: 1.19.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-19 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -200,6 +199,76 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: zstd-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: uri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: async-http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.86'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.86'
+- !ruby/object:Gem::Dependency
+  name: io-event
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.11.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.11.0
+- !ruby/object:Gem::Dependency
+  name: io-stream
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
 - !ruby/object:Gem::Dependency
   name: base64
   requirement: !ruby/object:Gem::Requirement
@@ -389,33 +458,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '4'
 - !ruby/object:Gem::Dependency
-  name: async
+  name: console
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.23'
+        version: '1.30'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.23'
-- !ruby/object:Gem::Dependency
-  name: async-http
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.50.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.50.0
+        version: '1.30'
 - !ruby/object:Gem::Dependency
   name: aws-sigv4
   requirement: !ruby/object:Gem::Requirement
@@ -477,6 +532,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".deepsource.toml"
+- ".rubocop.yml"
 - ADOPTERS.md
 - AUTHORS
 - CHANGELOG.md
@@ -671,6 +727,7 @@ files:
 - lib/fluent/plugin/metrics.rb
 - lib/fluent/plugin/metrics_local.rb
 - lib/fluent/plugin/multi_output.rb
+- lib/fluent/plugin/out_buffer.rb
 - lib/fluent/plugin/out_copy.rb
 - lib/fluent/plugin/out_exec.rb
 - lib/fluent/plugin/out_exec_filter.rb
@@ -725,9 +782,6 @@ files:
 - lib/fluent/plugin_helper/formatter.rb
 - lib/fluent/plugin_helper/http_server.rb
 - lib/fluent/plugin_helper/http_server/app.rb
-- lib/fluent/plugin_helper/http_server/compat/server.rb
-- lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb
-- lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb
 - lib/fluent/plugin_helper/http_server/methods.rb
 - lib/fluent/plugin_helper/http_server/request.rb
 - lib/fluent/plugin_helper/http_server/router.rb
@@ -752,6 +806,7 @@ files:
 - lib/fluent/registry.rb
 - lib/fluent/root_agent.rb
 - lib/fluent/rpc.rb
+- lib/fluent/source_only_buffer_agent.rb
 - lib/fluent/static_config_analysis.rb
 - lib/fluent/supervisor.rb
 - lib/fluent/system_config.rb
@@ -809,8 +864,11 @@ files:
 homepage: https://www.fluentd.org/
 licenses:
 - Apache-2.0
-metadata: {}
-post_install_message:
+metadata:
+  homepage_uri: https://www.fluentd.org/
+  source_code_uri: https://github.com/fluent/fluentd
+  changelog_uri: https://github.com/fluent/fluentd/blob/master/CHANGELOG.md
+  bug_tracker_uri: https://github.com/fluent/fluentd/issues
 rdoc_options: []
 require_paths:
 - lib
@@ -818,15 +876,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
-signing_key:
+rubygems_version: 3.6.8
 specification_version: 4
 summary: Fluentd event collector
 test_files: []

data/lib/fluent/plugin_helper/http_server/compat/server.rb

@@ -1,92 +0,0 @@
-#
-# Fluentd
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
-require 'fluent/plugin_helper/http_server/methods'
-require 'fluent/plugin_helper/http_server/compat/webrick_handler'
-require 'fluent/plugin_helper/http_server/compat/ssl_context_extractor'
-
-module Fluent
-  module PluginHelper
-    module HttpServer
-      module Compat
-        class Server
-          # @param logger [Logger]
-          # @param default_app [Object] ignored option. only for compat
-          # @param tls_context [OpenSSL::SSL::SSLContext]
-          def initialize(addr:, port:, logger:, default_app: nil, tls_context: nil)
-            @addr = addr
-            @port = port
-            @logger = logger
-
-            config = {
-              BindAddress: @addr,
-              Port: @port,
-              Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
-              AccessLog: [],
-            }
-            if tls_context
-              require 'webrick/https'
-              @logger.warn('Webrick ignores given TLS version')
-              tls_opt = Fluent::PluginHelper::HttpServer::Compat::SSLContextExtractor.extract(tls_context)
-              config = tls_opt.merge(**config)
-            end
-
-            @server = WEBrick::HTTPServer.new(config)
-
-            # @example ["/example.json", :get, handler object]
-            @methods = []
-
-            if block_given?
-              yield(self)
-            end
-          end
-
-          def start(notify = nil)
-            build_handler
-            notify.push(:ready)
-            @logger.debug('Start webrick HTTP server listening')
-            @server.start
-          end
-
-          def stop
-            @server.shutdown
-            @server.stop
-          end
-
-          HttpServer::Methods::ALL.map { |e| e.downcase.to_sym }.each do |name|
-            define_method(name) do |path, app = nil, &block|
-              if (block && app) || (!block && !app)
-                raise 'You must specify either app or block in the same time'
-              end
-
-              # Do not build a handler class here to able to handle multiple methods for single path.
-              @methods << [path, name, app || block]
-            end
-          end
-
-          private
-
-          def build_handler
-            @methods.group_by(&:first).each do |(path, rest)|
-              klass = Fluent::PluginHelper::HttpServer::Compat::WebrickHandler.build(**Hash[rest.map { |e| [e[1], e[2]] }])
-              @server.mount(path, klass)
-            end
-          end
-        end
-      end
-    end
-  end
-end