fluentd 1.18.0 → 1.19.0

data/lib/fluent/plugin_helper/storage.rb

@@ -38,7 +38,7 @@ module Fluent
         end
 
         s = @_storages[usage]
-        if s && s.running
+        if s&.running
           return s.storage
         elsif s
           # storage is already created, but not loaded / started

data/lib/fluent/plugin_id.rb

@@ -30,7 +30,7 @@ module Fluent
     end
 
     def configure(conf)
-      @_plugin_id_variable_store = Fluent::VariableStore.fetch_or_build(:pluing_id, default_value: Set.new)
+      @_plugin_id_variable_store = Fluent::VariableStore.fetch_or_build(:plugin_id, default_value: Set.new)
       @id = conf['@id']
       @_id_configured = !!@id # plugin id is explicitly configured by users (or not)
       if @id
@@ -57,13 +57,13 @@ module Fluent
     end
 
     def plugin_id_configured?
-      if instance_variable_defined?("@_id_configured")
+      if instance_variable_defined?(:@_id_configured)
         @_id_configured
       end
     end
 
     def plugin_id
-      if instance_variable_defined?("@id")
+      if instance_variable_defined?(:@id)
         @id || "object:#{object_id.to_s(16)}"
       else
         "object:#{object_id.to_s(16)}"

data/lib/fluent/root_agent.rb

@@ -86,7 +86,7 @@ module Fluent
       @without_source = system_config.without_source || false
       @source_only_mode = SourceOnlyMode.new(system_config.with_source_only, start_in_parallel)
       @source_only_buffer_agent = nil
-      @enable_input_metrics = system_config.enable_input_metrics || false
+      @enable_input_metrics = system_config.enable_input_metrics
 
       suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
     end
@@ -103,6 +103,7 @@ module Fluent
       used_worker_ids = []
       available_worker_ids = (0..Fluent::Engine.system_config.workers - 1).to_a
       # initialize <worker> elements
+      supported_directives = ['source', 'match', 'filter', 'label']
       conf.elements(name: 'worker').each do |e|
         target_worker_id_str = e.arg
         if target_worker_id_str.empty?
@@ -131,7 +132,7 @@ module Fluent
             used_worker_ids << target_worker_id
 
             e.elements.each do |elem|
-              unless ['source', 'match', 'filter', 'label'].include?(elem.name)
+              unless supported_directives.include?(elem.name)
                 raise Fluent::ConfigError, "<worker> section cannot have <#{elem.name}> directive"
               end
             end
@@ -147,7 +148,7 @@ module Fluent
           end
 
           e.elements.each do |elem|
-            unless ['source', 'match', 'filter', 'label'].include?(elem.name)
+            unless supported_directives.include?(elem.name)
               raise Fluent::ConfigError, "<worker> section cannot have <#{elem.name}> directive"
             end
             elem.set_target_worker_id(target_worker_id)

data/lib/fluent/static_config_analysis.rb

@@ -18,7 +18,7 @@ require 'fluent/config'
 require 'fluent/plugin'
 
 module Fluent
-  # Static Analysis means analysing all plugins and Fluent::Element without invokeing Plugin#configure
+  # Static Analysis means analysing all plugins and Fluent::Element without invoking Plugin#configure
   class StaticConfigAnalysis
     module Elem
       Input = Struct.new(:plugin, :config)
@@ -72,6 +72,7 @@ module Fluent
       available_worker_ids = [*0...@workers]
 
       ret = []
+      supported_directives = %w[source match filter label]
       conf.elements(name: 'worker').each do |config|
         ids = parse_worker_id(config)
         ids.each do |id|
@@ -83,7 +84,7 @@ module Fluent
         end
 
         config.elements.each do |elem|
-          unless %w[source match filter label].include?(elem.name)
+          unless supported_directives.include?(elem.name)
             raise Fluent::ConfigError, "<worker> section cannot have <#{elem.name}> directive"
           end
         end

data/lib/fluent/supervisor.rb

@@ -17,6 +17,7 @@
 require 'fileutils'
 require 'open3'
 require 'pathname'
+require 'find'
 
 require 'fluent/config'
 require 'fluent/counter'
@@ -587,7 +588,7 @@ module Fluent
         log_level: params['log_level'],
         chuser: params['chuser'],
         chgroup: params['chgroup'],
-        chumask: params['chumask'],
+        chumask: params['chumask'].is_a?(Integer) ? params['chumask'] : params['chumask']&.to_i(8),
         daemonize: daemonize,
         rpc_endpoint: params['rpc_endpoint'],
         counter_server: params['counter_server'],
@@ -630,7 +631,7 @@ module Fluent
         ignore_repeated_log_interval: nil,
         without_source: nil,
         with_source_only: nil,
-        enable_input_metrics: nil,
+        enable_input_metrics: true,
         enable_size_metrics: nil,
         use_v1_config: true,
         strict_config_value: nil,
@@ -721,6 +722,18 @@ module Fluent
         $log.error 'config error', file: @config_path, error: e
         $log.debug_backtrace
         exit!(1)
+      rescue ScriptError => e # LoadError, NotImplementedError, SyntaxError
+        if e.respond_to?(:path)
+          $log.error e.message, path: e.path, error: e
+        else
+          $log.error e.message, error: e
+        end
+        $log.debug_backtrace
+        exit!(1)
+      rescue => e
+        $log.error "unexpected error", error: e
+        $log.debug_backtrace
+        exit!(1)
       end
 
       if dry_run
@@ -754,7 +767,7 @@ module Fluent
 
       install_main_process_signal_handlers
 
-      # This is the only log messsage for @standalone_worker
+      # This is the only log message for @standalone_worker
       $log.info "starting fluentd-#{Fluent::VERSION} without supervision", pid: Process.pid, ruby: RUBY_VERSION if @standalone_worker
 
       main_process do
@@ -794,6 +807,10 @@ module Fluent
 
       $log.info :supervisor, 'parsing config file is succeeded', path: @config_path
 
+      build_additional_configurations do |additional_conf|
+        @conf += additional_conf
+      end
+
       @libs.each do |lib|
         require lib
       end
@@ -891,11 +908,12 @@ module Fluent
         ignore_repeated_log_interval: system_config.ignore_repeated_log_interval,
         ignore_same_log_interval: system_config.ignore_same_log_interval,
       )
+      $log.force_stacktrace_level(system_config.log.forced_stacktrace_level) if system_config.force_stacktrace_level?
       $log.enable_color(false) if actual_log_path
       $log.enable_debug if system_config.log_level <= Fluent::Log::LEVEL_DEBUG
 
       $log.info "init #{process_type} logger",
-                path: actual_log_path, 
+                path: actual_log_path,
                 rotate_age: @log_rotate_age,
                 rotate_size: @log_rotate_size
     end
@@ -924,6 +942,7 @@ module Fluent
         'inline_config' => @inline_config,
         'chuser' => @chuser,
         'chgroup' => @chgroup,
+        'chumask' => @chumask,
         'fluentd_conf_path' => @config_path,
         'fluentd_conf' => @conf.to_s,
         'use_v1_config' => @use_v1_config,
@@ -1077,6 +1096,10 @@ module Fluent
             type: @config_file_type,
           )
 
+          build_additional_configurations do |additional_conf|
+            conf += additional_conf
+          end
+
           Fluent::VariableStore.try_to_reset do
             Fluent::Engine.reload_config(conf)
           end
@@ -1168,7 +1191,7 @@ module Fluent
     def build_system_config(conf)
       system_config = SystemConfig.create(conf, @cl_opt[:strict_config_value])
       # Prefer the options explicitly specified in the command line
-      # 
+      #
       # TODO: There is a bug that `system_config.log.rotate_age/rotate_size` are
       # not merged with the command line options since they are not in
       # `SYSTEM_CONFIG_PARAMETERS`.
@@ -1183,6 +1206,29 @@ module Fluent
       system_config
     end
 
+    def build_additional_configurations
+      if @system_config.config_include_dir&.empty?
+        $log.info :supervisor, 'configuration include directory is disabled'
+        return
+      end
+      begin
+        supported_suffixes = [".conf", ".yaml", ".yml"]
+        Find.find(@system_config.config_include_dir) do |path|
+          next if File.directory?(path)
+          next unless supported_suffixes.include?(File.extname(path))
+          # NOTE: both types of normal config (.conf) and YAML will be loaded.
+          # Thus, it does not care whether @config_path is .conf or .yml.
+          $log.info :supervisor, 'loading additional configuration file', path: path
+          yield Fluent::Config.build(config_path: path,
+                                     encoding: @conf_encoding,
+                                     use_v1_config: @use_v1_config,
+                                     type: :guess)
+        end
+      rescue Errno::ENOENT
+        $log.info :supervisor, 'inaccessible include directory was specified', path: @system_config.config_include_dir
+      end
+    end
+
     RUBY_ENCODING_OPTIONS_REGEX = %r{\A(-E|--encoding=|--internal-encoding=|--external-encoding=)}.freeze
 
     def build_spawn_command

data/lib/fluent/system_config.rb

@@ -16,6 +16,7 @@
 
 require 'fluent/configurable'
 require 'fluent/config/element'
+require 'fluent/env'
 
 module Fluent
   class SystemConfig
@@ -28,7 +29,8 @@ module Fluent
       :without_source, :with_source_only, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
       :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket,
-      :metrics, :enable_input_metrics, :enable_size_metrics, :enable_jit, :source_only_buffer
+      :metrics, :enable_input_metrics, :enable_size_metrics, :enable_jit, :source_only_buffer,
+      :config_include_dir
     ]
 
     config_param :workers,   :integer, default: 1
@@ -49,7 +51,7 @@ module Fluent
     config_param :strict_config_value, :bool, default: nil
     config_param :enable_msgpack_time_support, :bool, default: nil
     config_param :disable_shared_socket, :bool, default: nil
-    config_param :enable_input_metrics, :bool, default: nil
+    config_param :enable_input_metrics, :bool, default: true
     config_param :enable_size_metrics, :bool, default: nil
     config_param :enable_jit, :bool, default: false
     config_param :file_permission, default: nil do |v|
@@ -58,6 +60,7 @@ module Fluent
     config_param :dir_permission, default: nil do |v|
       v.to_i(8)
     end
+    config_param :config_include_dir, default: Fluent::DEFAULT_CONFIG_INCLUDE_DIR
     config_section :log, required: false, init: true, multi: false do
       config_param :path, :string, default: nil
       config_param :format, :enum, list: [:text, :json], default: :text
@@ -76,6 +79,7 @@ module Fluent
         end
       end
       config_param :rotate_size, :size, default: nil
+      config_param :forced_stacktrace_level, :enum, list: [:none, :trace, :debug, :info, :warn, :error, :fatal], default: :none
     end
 
     config_section :counter_server, multi: false do
@@ -115,6 +119,10 @@ module Fluent
       config_param :compress, :enum, list: [:text, :gzip], default: nil
     end
 
+    def force_stacktrace_level?
+      @log.forced_stacktrace_level != :none
+    end
+
     def self.create(conf, strict_config_value=false)
       systems = conf.elements(name: 'system')
       return SystemConfig.new if systems.empty?
@@ -159,6 +167,7 @@ module Fluent
       end
 
       @log_level = Log.str_to_level(@log_level.to_s) if @log_level
+      @log[:forced_stacktrace_level] = Log.str_to_level(@log.forced_stacktrace_level.to_s) if force_stacktrace_level?
     end
 
     def dup
@@ -183,13 +192,13 @@ module Fluent
         unless defined?($_system_config)
           $_system_config = nil
         end
-        (instance_variable_defined?("@_system_config") && @_system_config) ||
+        (instance_variable_defined?(:@_system_config) && @_system_config) ||
           $_system_config || Fluent::Engine.system_config
       end
 
       def system_config_override(opts={})
         require 'fluent/engine'
-        if !instance_variable_defined?("@_system_config") || @_system_config.nil?
+        if !instance_variable_defined?(:@_system_config) || @_system_config.nil?
           @_system_config = (defined?($_system_config) && $_system_config ? $_system_config : Fluent::Engine.system_config).dup
         end
         opts.each_pair do |key, value|

data/lib/fluent/test/base.rb

@@ -32,7 +32,7 @@ module Fluent
             #   klass.dup is worse because its ancestors does NOT include original class name
             klass_name = klass.name
             klass = Class.new(klass)
-            klass.define_singleton_method("name") { klass_name }
+            klass.define_singleton_method(:name) { klass_name }
             klass.module_eval(&block)
           end
           @instance = klass.new

data/lib/fluent/test/driver/base.rb

@@ -185,7 +185,7 @@ module Fluent
           end
         end
 
-        def run_actual(timeout: DEFAULT_TIMEOUT, &block)
+        def run_actual(timeout: DEFAULT_TIMEOUT)
           if @instance.respond_to?(:_threads)
             sleep 0.1 until @instance._threads.values.all?(&:alive?)
           end
@@ -201,7 +201,7 @@ module Fluent
           return_value = nil
           begin
             Timeout.timeout(timeout * 2) do |sec|
-              return_value = block.call if block_given?
+              return_value = yield if block_given?
             end
           rescue Timeout::Error
             raise TestTimedOut, "Test case timed out with hard limit."

data/lib/fluent/test/filter_test.rb

@@ -59,9 +59,9 @@ module Fluent
       alias_method :emits, :filtered_as_array # emits is for consistent with other drivers
 
       # Almost filters don't use threads so default is 0. It reduces test time.
-      def run(num_waits = 0, &block)
+      def run(num_waits = 0)
         super(num_waits) {
-          block.call if block
+          yield if block_given?
 
           @events.each { |tag, es|
             processed = @instance.filter_stream(tag, es)

data/lib/fluent/test/formatter_test.rb

@@ -28,7 +28,7 @@ module Fluent
             #   klass.dup is worse because its ancestors does NOT include original class name
             klass_name = klass_or_str.name
             klass_or_str = Class.new(klass_or_str)
-            klass_or_str.define_singleton_method("name") { klass_name }
+            klass_or_str.define_singleton_method(:name) { klass_name }
             klass_or_str.module_eval(&block)
           end
           @instance = klass_or_str.new

data/lib/fluent/test/helpers.rb

@@ -50,6 +50,10 @@ EOT
         end
       end
 
+      def event_time_without_nsec(str=nil, format: nil)
+        Fluent::EventTime.new(event_time(str, format: format))
+      end
+
       def with_timezone(tz)
         oldtz, ENV['TZ'] = ENV['TZ'], tz
         yield

data/lib/fluent/test/input_test.rb

@@ -109,7 +109,7 @@ module Fluent
         end
       end
 
-      def run(num_waits = 10, &block)
+      def run(num_waits = 10)
         m = method(:emit_stream)
         unless Engine.singleton_class.ancestors.include?(EmitStreamWrapper)
           Engine.singleton_class.prepend EmitStreamWrapper
@@ -121,7 +121,7 @@ module Fluent
         instance.router.emit_stream_callee = m
 
         super(num_waits) {
-          block.call if block
+          yield if block_given?
 
           if @expected_emits_length || @expects || @run_post_conditions
             # counters for emits and emit_streams

data/lib/fluent/test/output_test.rb

@@ -71,10 +71,10 @@ module Fluent
         (@expected_buffer ||= '') << str
       end
 
-      def run(num_waits = 10, &block)
+      def run(num_waits = 10)
         result = nil
         super(num_waits) {
-          block.call if block
+          yield if block_given?
 
           es = ArrayEventStream.new(@entries)
           buffer = @instance.format_stream(@tag, es)
@@ -119,10 +119,10 @@ module Fluent
         (@expected_buffer ||= '') << str
       end
 
-      def run(&block)
+      def run
         result = []
         super {
-          block.call if block
+          yield if block_given?
 
           buffer = ''
           lines = {}

data/lib/fluent/test/parser_test.rb

@@ -27,7 +27,7 @@ module Fluent
             #   klass.dup is worse because its ancestors does NOT include original class name
             klass_name = klass_or_str.name
             klass_or_str = Class.new(klass_or_str)
-            klass_or_str.define_singleton_method("name") { klass_name }
+            klass_or_str.define_singleton_method(:name) { klass_name }
             klass_or_str.module_eval(&block)
           end
           case klass_or_str.instance_method(:initialize).arity

data/lib/fluent/tls.rb

@@ -76,6 +76,30 @@ module Fluent
       ctx
     end
     module_function :set_version_to_context
+
+    def set_version_to_options(opt, version, min_version, max_version)
+      if MIN_MAX_AVAILABLE
+        case
+        when min_version.nil? && max_version.nil?
+          min_version = METHODS_MAP[version] || version
+          max_version = METHODS_MAP[version] || version
+        when min_version.nil? && max_version
+          raise Fluent::ConfigError, "When you set max_version, must set min_version together"
+        when min_version && max_version.nil?
+          raise Fluent::ConfigError, "When you set min_version, must set max_version together"
+        else
+          min_version = METHODS_MAP[min_version] || min_version
+          max_version = METHODS_MAP[max_version] || max_version
+        end
+        opt[:min_version] = min_version
+        opt[:max_version] = max_version
+      else
+        opt[:ssl_version] = METHODS_MAP[version] || version
+      end
+
+      opt
+    end
+    module_function :set_version_to_options
   end
 end
 

data/lib/fluent/variable_store.rb

@@ -15,7 +15,7 @@
 #
 
 module Fluent
-  # VariableStore provides all pluigns with the way to shared variable without using class variable
+  # VariableStore provides all plugins with the way to shared variable without using class variable
   # it's for safe reloading mechanism
   class VariableStore
     @data = {}

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.18.0'
+  VERSION = '1.19.0'
 
 end

data/lib/fluent/winsvc.rb

@@ -32,7 +32,7 @@ begin
   op.parse(ARGV)
   if opts[:service_name] == nil
     raise "Error: No Windows Service name set. Use '--service-name'"
-  end 
+  end
 
   def read_fluentdopt(service_name)
     Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\#{service_name}") do |reg|
@@ -48,25 +48,32 @@ begin
   end
 
   class FluentdService < Daemon
+    ERROR_WAIT_NO_CHILDREN = 128
+
     @pid = 0
     @service_name = ''
 
     def initialize(service_name)
       @service_name = service_name
     end
-    
+
     def service_main
       @pid = service_main_start(@service_name)
-      while running?
-        sleep 10
+      begin
+        loop do
+          sleep 5
+          break unless running?
+          raise Errno::ECHILD if Process.waitpid(@pid, Process::WNOHANG)
+        end
+      rescue Errno::ECHILD
+        @pid = 0
+        SetEvent(@@hStopEvent)
+        SetTheServiceStatus.call(SERVICE_STOPPED, ERROR_WAIT_NO_CHILDREN, 0, 0)
       end
     end
 
     def service_stop
-      set_event(@service_name)
-      if @pid > 0
-        Process.waitpid(@pid)
-      end
+      wait_supervisor_finished if @pid > 0
     end
 
     def service_paramchange
@@ -91,6 +98,29 @@ begin
       ev.set
       ev.close
     end
+
+    def repeat_set_event_several_times_until_success(event_name)
+      retries = 0
+      max_retries = 10
+      delay_sec = 3
+
+      begin
+        set_event(event_name)
+      rescue Errno::ENOENT
+        # This error occurs when the supervisor process has not yet created the event.
+        # If STOP is immediately executed, this state will occur.
+        # Retry `set_event' to wait for the initialization of the supervisor.
+        retries += 1
+        raise if max_retries < retries
+        sleep(delay_sec)
+        retry
+      end
+    end
+
+    def wait_supervisor_finished
+      repeat_set_event_several_times_until_success(@service_name)
+      Process.waitpid(@pid)
+    end
   end
 
   FluentdService.new(opts[:service_name]).mainloop