fluentd 1.8.1 → 1.9.0.rc1

data/lib/fluent/plugin/in_http.rb

@@ -331,29 +331,29 @@ module Fluent::Plugin
         headers.each_pair {|k,v|
           @env["HTTP_#{k.gsub('-','_').upcase}"] = v
           case k
-          when /Expect/i
+          when /\AExpect\z/i
             expect = v
-          when /Content-Length/i
+          when /\AContent-Length\Z/i
             size = v.to_i
-          when /Content-Type/i
+          when /\AContent-Type\Z/i
             @content_type = v
-          when /Content-Encoding/i
+          when /\AContent-Encoding\Z/i
             @content_encoding = v
-          when /Connection/i
+          when /\AConnection\Z/i
             if v =~ /close/i
               @keep_alive = false
             elsif v =~ /Keep-alive/i
               @keep_alive = true
             end
-          when /Origin/i
+          when /\AOrigin\Z/i
             @origin  = v
-          when /X-Forwarded-For/i
+          when /\AX-Forwarded-For\Z/i
             # For multiple X-Forwarded-For headers. Use first header value.
             v = v.first if v.is_a?(Array)
             @remote_addr = v.split(",").first
-          when /Access-Control-Request-Method/i
+          when /\AAccess-Control-Request-Method\Z/i
             @access_control_request_method = v
-          when /Access-Control-Request-Headers/i
+          when /\AAccess-Control-Request-Headers\Z/i
             @access_control_request_headers = v
           end
         }

data/lib/fluent/plugin/in_tail.rb

@@ -21,6 +21,7 @@ require 'fluent/config/error'
 require 'fluent/event'
 require 'fluent/plugin/buffer'
 require 'fluent/plugin/parser_multiline'
+require 'fluent/variable_store'
 
 if Fluent.windows?
   require_relative 'file_wrapper'
@@ -105,9 +106,8 @@ module Fluent::Plugin
 
     attr_reader :paths
 
-    @@pos_file_paths = {}
-
     def configure(conf)
+      @variable_store = Fluent::VariableStore.fetch_or_build(:in_tail)
       compat_parameters_convert(conf, :parser)
       parser_config = conf.elements('parse').first
       unless parser_config
@@ -139,11 +139,11 @@ module Fluent::Plugin
 
       # TODO: Use plugin_root_dir and storage plugin to store positions if available
       if @pos_file
-        if @@pos_file_paths.has_key?(@pos_file) && !called_in_test?
-          plugin_id_using_this_path = @@pos_file_paths[@pos_file]
+        if @variable_store.key?(@pos_file) && !called_in_test?
+          plugin_id_using_this_path = @variable_store[@pos_file]
           raise Fluent::ConfigError, "Other 'in_tail' plugin already use same pos_file path: plugin_id = #{plugin_id_using_this_path}, pos_file path = #{@pos_file}"
         end
-        @@pos_file_paths[@pos_file] = self.plugin_id
+        @variable_store[@pos_file] = self.plugin_id
       else
         $log.warn "'pos_file PATH' parameter is not set to a 'tail' source."
         $log.warn "this parameter is highly recommended to save the position to resume tailing."
@@ -212,7 +212,9 @@ module Fluent::Plugin
     end
 
     def stop
-      @@pos_file_paths.delete(@pos_file)
+      if @variable_store
+        @variable_store.delete(@pos_file)
+      end
 
       super
     end

data/lib/fluent/plugin/out_http.rb

@@ -212,9 +212,9 @@ module Fluent::Plugin
             end
 
       if res.is_a?(Net::HTTPSuccess)
-        log.debug { "#{res.code} #{res.message}#{res.body}" }
+        log.debug { "#{res.code} #{res.message.rstrip}#{res.body.lstrip}" }
       else
-        msg = "#{res.code} #{res.message}#{res.body}"
+        msg = "#{res.code} #{res.message.rstrip} #{res.body.lstrip}"
 
         if @retryable_response_codes.include?(res.code.to_i)
           raise RetryableResponse, msg

data/lib/fluent/plugin/output.rb

@@ -387,7 +387,7 @@ module Fluent
           @secondary.acts_as_secondary(self)
           @secondary.configure(secondary_conf)
           if (self.class != @secondary.class) && (@custom_format || @secondary.implement?(:custom_format))
-            log.warn "secondary type should be same with primary one", primary: self.class.to_s, secondary: @secondary.class.to_s
+            log.warn "Use different plugin for secondary. Check the plugin works with primary like secondary_file", primary: self.class.to_s, secondary: @secondary.class.to_s
           end
         else
           @secondary = nil

data/lib/fluent/plugin/parser.rb

@@ -105,6 +105,12 @@ module Fluent
         :text_per_line
       end
 
+      def initialize
+        super
+
+        @timeout_checker = nil
+      end
+
       def configure(conf)
         super
 

data/lib/fluent/plugin_helper/http_server.rb

@@ -15,7 +15,6 @@
 #
 
 begin
-  # raise if RUBY_VERSION < 2.3.x. see Gemfile
   require 'async'
   require 'fluent/plugin_helper/http_server/server'
 rescue LoadError => _

data/lib/fluent/plugin_helper/record_accessor.rb

@@ -15,14 +15,6 @@
 #
 
 require 'fluent/config/error'
-unless {}.respond_to?(:dig)
-  begin
-    # backport_dig is faster than dig_rb so prefer backport_dig.
-    require 'backport_dig'
-  rescue LoadError
-    require 'dig_rb'
-  end
-end
 
 module Fluent
   module PluginHelper

data/lib/fluent/plugin_helper/server.rb

@@ -693,26 +693,11 @@ module Fluent
             end
           end
 
-          if RUBY_VERSION.to_f >= 2.3
-            NONBLOCK_ARG = { exception: false }
-            def try_handshake
-              @_handler_socket.accept_nonblock(**NONBLOCK_ARG)
-            end
-          else
-            def try_handshake
-              @_handler_socket.accept_nonblock
-            rescue IO::WaitReadable
-              :wait_readable
-            rescue IO::WaitWritable
-              :wait_writable
-            end
-          end
-
           def try_tls_accept
             return true if @_handler_accepted
 
             begin
-              result = try_handshake # this method call actually try to do handshake via TLS
+              result = @_handler_socket.accept_nonblock(exception: false) # this method call actually try to do handshake via TLS
               if result == :wait_readable || result == :wait_writable
                 # retry accept_nonblock: there aren't enough data in underlying socket buffer
               else

data/lib/fluent/plugin_id.rb

@@ -15,26 +15,29 @@
 #
 
 require 'set'
+require 'fluent/variable_store'
 
 module Fluent
   module PluginId
-    @@configured_ids = Set.new
 
     def initialize
       super
+
+      @_plugin_id_variable_store = nil
       @_plugin_root_dir = nil
       @id = nil
     end
 
     def configure(conf)
+      @_plugin_id_variable_store = Fluent::VariableStore.fetch_or_build(:pluing_id, default_value: Set.new)
       @id = conf['@id']
       @_id_configured = !!@id # plugin id is explicitly configured by users (or not)
       if @id
         @id = @id.to_s
-        if @@configured_ids.include?(@id) && !plugin_id_for_test?
+        if @_plugin_id_variable_store.include?(@id) && !plugin_id_for_test?
           raise Fluent::ConfigError, "Duplicated plugin id `#{@id}`. Check whole configuration and fix it."
         end
-        @@configured_ids.add(@id)
+        @_plugin_id_variable_store.add(@id)
       end
 
       super
@@ -79,7 +82,9 @@ module Fluent
     end
 
     def stop
-      @@configured_ids.delete(@id)
+      if @_plugin_id_variable_store
+        @_plugin_id_variable_store.delete(@id)
+      end
 
       super
     end

data/lib/fluent/static_config_analysis.rb

@@ -0,0 +1,194 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config'
+require 'fluent/plugin'
+
+module Fluent
+  # Static Analysis means analysing all plugins and Fluent::Element without invokeing Plugin#configure
+  class StaticConfigAnalysis
+    module Elem
+      Input = Struct.new(:plugin, :config)
+      Output = Struct.new(:plugin, :config)
+      Filter = Struct.new(:plugin, :config)
+      Label = Struct.new(:name, :config, :nodes)
+      Worker = Struct.new(:ids, :config, :nodes)
+    end
+
+    Result = Struct.new(:tree, :outputs, :inputs, :filters, :labels) do
+      def all_plugins
+        (outputs + inputs + filters).map(&:plugin)
+      end
+    end
+
+    # @param workers [Integer] Number of workers
+    # @return [Fluent::StaticConfigAnalysis::Result]
+    def self.call(conf, workers: 1)
+      new(workers).call(conf)
+    end
+
+    def initialize(workers)
+      @workers = workers
+
+      reset
+    end
+
+    def call(config)
+      reset
+
+      tree = [
+        static_worker_analyse(config),
+        static_label_analyse(config),
+        static_filter_and_output_analyse(config),
+        static_input_analyse(config),
+      ].flatten
+
+      Result.new(tree, @outputs, @inputs, @filters, @labels.values)
+    end
+
+    private
+
+    def reset
+      @outputs = []
+      @inputs = []
+      @filters = []
+      @labels = {}
+    end
+
+    def static_worker_analyse(conf)
+      available_worker_ids = [*0...@workers]
+
+      ret = []
+      conf.elements(name: 'worker').each do |config|
+        ids = parse_worker_id(config)
+        ids.each do |id|
+          if available_worker_ids.include?(id)
+            available_worker_ids.delete(id)
+          else
+            raise Fluent::ConfigError, "specified worker_id<#{id}> collisions is detected on <worker> directive. Available worker id(s): #{available_worker_ids}"
+          end
+        end
+
+        config.elements.each do |elem|
+          unless %w[source match filter label].include?(elem.name)
+            raise Fluent::ConfigError, "<worker> section cannot have <#{elem.name}> directive"
+          end
+        end
+
+        nodes = [
+          static_label_analyse(config),
+          static_filter_and_output_analyse(config),
+          static_input_analyse(config),
+        ].flatten
+        ret << Elem::Worker.new(ids, config, nodes)
+      end
+
+      ret
+    end
+
+    def parse_worker_id(conf)
+      worker_id_str = conf.arg
+
+      if worker_id_str.empty?
+        raise Fluent::ConfigError, 'Missing worker id on <worker> directive'
+      end
+
+      l, r =
+         begin
+           worker_id_str.split('-', 2).map { |v| Integer(v) }
+         rescue TypeError, ArgumentError
+           raise Fluent::ConfigError, "worker id should be integer: #{worker_id_str}"
+         end
+
+      if l < 0 || l >= @workers
+        raise Fluent::ConfigError, "worker id #{l} specified by <worker> directive is not allowed. Available worker id is between 0 and #{@workers-1}"
+      end
+
+      # e.g. specified one worker id like `<worker 0>`
+      if r.nil?
+        return [l]
+      end
+
+      if r < 0 || r >= @workers
+        raise Fluent::ConfigError, "worker id #{r} specified by <worker> directive is not allowed. Available worker id is between 0 and #{@workers-1}"
+      end
+
+      if l > r
+        raise Fluent::ConfigError, "greater first_worker_id<#{l}> than last_worker_id<#{r}> specified by <worker> directive is not allowed. Available multi worker assign syntax is <smaller_worker_id>-<greater_worker_id>"
+      end
+
+      [l, r]
+    end
+
+    def static_label_analyse(conf)
+      ret = []
+      conf.elements(name: 'label').each do |e|
+        name = e.arg
+        if name.empty?
+          raise ConfigError, 'Missing symbol argument on <label> directive'
+        end
+
+        if @labels[name]
+          raise ConfigError, "Section <label #{name}> appears twice"
+        end
+
+        l = Elem::Label.new(name, e, static_filter_and_output_analyse(e))
+        ret << l
+        @labels[name] = l
+      end
+
+      ret
+    end
+
+    def static_filter_and_output_analyse(conf)
+      ret = []
+      conf.elements('filter', 'match').each do |e|
+        type = e['@type']
+        if type.nil? || type.empty?
+          raise Fluent::ConfigError, "Missing '@type' parameter on <#{e.name}> directive"
+        end
+
+        if e.name == 'filter'
+          f = Elem::Filter.new(Fluent::Plugin.new_filter(type), e)
+          ret << f
+          @filters << f
+        else
+          o = Elem::Output.new(Fluent::Plugin.new_output(type), e)
+          ret << o
+          @outputs << o
+        end
+      end
+
+      ret
+    end
+
+    def static_input_analyse(conf)
+      ret = []
+      conf.elements(name: 'source').each do |e|
+        type = e['@type']
+        if type.nil? || type.empty?
+          raise Fluent::ConfigError, "Missing '@type' parameter on <#{e.name}> directive"
+        end
+
+        i = Elem::Input.new(Fluent::Plugin.new_input(type), e)
+        @inputs << i
+        ret << i
+      end
+
+      ret
+    end
+  end
+end

data/lib/fluent/supervisor.rb

@@ -26,6 +26,7 @@ require 'fluent/plugin'
 require 'fluent/rpc'
 require 'fluent/system_config'
 require 'fluent/msgpack_factory'
+require 'fluent/variable_store'
 require 'serverengine'
 
 if Fluent.windows?
@@ -42,6 +43,7 @@ end
 module Fluent
   module ServerModule
     def before_run
+      @fluentd_conf = config[:fluentd_conf]
       @rpc_server = nil
       @counter = nil
 
@@ -120,6 +122,15 @@ module Fluent
         nil
       }
 
+      @rpc_server.mount_proc('/api/config.gracefulReload') { |req, res|
+        $log.debug "fluentd RPC got /api/config.gracefulReload request"
+        unless Fluent.windows?
+          Process.kill :USR2, $$
+        end
+
+        nil
+      }
+
       @rpc_server.mount_proc('/api/config.getDump') { |req, res|
         $log.debug "fluentd RPC got /api/config.getDump request"
         $log.info "get dump in-memory config via HTTP"
@@ -156,6 +167,11 @@ module Fluent
         $log.debug "fluentd supervisor process get SIGUSR1"
         supervisor_sigusr1_handler
       end unless Fluent.windows?
+
+      trap :USR2 do
+        $log.debug 'fluentd supervisor process got SIGUSR2'
+        supervisor_sigusr2_handler
+      end unless Fluent.windows?
     end
 
     def install_windows_event_handler
@@ -179,20 +195,36 @@ module Fluent
     end
 
     def supervisor_sigusr1_handler
-      if log = config[:logger_initializer]
-        # Creating new thread due to mutex can't lock
-        # in main thread during trap context
-        Thread.new do
-          log.reopen!
-        end
-      end
+      reopen_log
+      send_signal_to_workers(:USR1)
+    end
 
-      if config[:worker_pid]
-        config[:worker_pid].each_value do |pid|
-          Process.kill(:USR1, pid)
-          # don't rescue Errno::ESRCH here (invalid status)
+    def supervisor_sigusr2_handler
+      conf = nil
+      t = Thread.new do
+        $log.info 'Reloading new config'
+
+        # Validate that loading config is valid at first
+        conf = Fluent::Config.build(
+          config_path: config[:config_path],
+          encoding: config[:conf_encoding],
+          additional_config: config[:inline_config],
+          use_v1_config: config[:use_v1_config],
+        )
+
+        Fluent::VariableStore.try_to_reset do
+          Fluent::Engine.reload_config(conf, supervisor: true)
         end
       end
+
+      t.report_on_exception = false # Error is handled by myself
+      t.join
+
+      reopen_log
+      send_signal_to_workers(:USR2)
+      @fluentd_conf = conf.to_s
+    rescue => e
+      $log.error "Failed to reload config file: #{e}"
     end
 
     def kill_worker
@@ -210,11 +242,32 @@ module Fluent
     end
 
     def supervisor_dump_config_handler
-      $log.info config[:fluentd_conf]
+      $log.info @fluentd_conf
     end
 
     def supervisor_get_dump_config_handler
-      {conf: config[:fluentd_conf]}
+      { conf: @fluentd_conf }
+    end
+
+    private
+
+    def reopen_log
+      if (log = config[:logger_initializer])
+        # Creating new thread due to mutex can't lock
+        # in main thread during trap context
+        Thread.new do
+          log.reopen!
+        end
+      end
+    end
+
+    def send_signal_to_workers(signal)
+      return unless config[:worker_pid]
+
+      config[:worker_pid].each_value do |pid|
+        # don't rescue Errno::ESRCH here (invalid status)
+        Process.kill(signal, pid)
+      end
     end
   end
 
@@ -302,6 +355,9 @@ module Fluent
                                  JSON.dump(params)],
         command_sender: command_sender,
         fluentd_conf: params['fluentd_conf'],
+        conf_encoding: params['conf_encoding'],
+        inline_config: params['inline_config'],
+        config_path: path,
         main_cmd: params['main_cmd'],
         signame: params['signame'],
       }
@@ -558,8 +614,7 @@ module Fluent
         $log.warn('the value "-" for `inline_config` is deprecated. See https://github.com/fluent/fluentd/issues/2711')
         @inline_config = STDIN.read
       end
-
-      @conf = read_config
+      @conf = Fluent::Config.build(config_path: @config_path, encoding: @conf_encoding, additional_config: @inline_config, use_v1_config: @use_v1_config)
       @system_config = build_system_config(@conf)
 
       @log.level = @system_config.log_level
@@ -674,6 +729,10 @@ module Fluent
         flush_buffer
       end unless Fluent.windows?
 
+      trap :USR2 do
+        reload_config
+      end unless Fluent.windows?
+
       if Fluent.windows?
         command_pipe = STDIN.dup
         STDIN.reopen(File::NULL, "rb")
@@ -714,6 +773,32 @@ module Fluent
       end
     end
 
+    def reload_config
+      Thread.new do
+        $log.debug('worker got SIGUSR2')
+
+        begin
+          conf = Fluent::Config.build(
+            config_path: @config_path,
+            encoding: @conf_encoding,
+            additional_config: @inline_config,
+            use_v1_config: @use_v1_config,
+          )
+
+          Fluent::VariableStore.try_to_reset do
+            Fluent::Engine.reload_config(conf)
+          end
+        rescue => e
+          # it is guranteed that config file is valid by supervisor side. but it's not atomic becuase of using signals to commnicate between worker and super
+          # So need this rescue code
+          $log.error("failed to reload config: #{e}")
+          next
+        end
+
+        @conf = conf
+      end
+    end
+
     def logging_with_console_output
       yield $log
       unless @log.stdout?
@@ -770,16 +855,6 @@ module Fluent
       exit!(unrecoverable_error ? 2 : 1)
     end
 
-    def read_config
-      config_fname = File.basename(@config_path)
-      config_basedir = File.dirname(@config_path)
-      config_data = File.open(@config_path, "r:#{@conf_encoding}:utf-8") {|f| f.read }
-      if @inline_config
-        config_data << "\n" << @inline_config.gsub("\\n", "\n")
-      end
-      Fluent::Config.parse(config_data, config_fname, config_basedir, @use_v1_config)
-    end
-
     def build_system_config(conf)
       system_config = SystemConfig.create(conf, @cl_opt[:strict_config_value])
       opt = {}
@@ -794,7 +869,7 @@ module Fluent
           opt[param] = @cl_opt[param]
         end
       end
-      system_config.overwrite_variables(opt)
+      system_config.overwrite_variables(**opt)
       system_config
     end
   end