fluentd 0.12.20 → 0.12.21

data/lib/fluent/process.rb

@@ -14,6 +14,12 @@
 #    limitations under the License.
 #
 
+require 'thread'
+
+require 'fluent/config'
+require 'fluent/engine'
+require 'fluent/event'
+
 module Fluent
   class DetachProcessManager
     require 'singleton'
@@ -149,12 +155,12 @@ module Fluent
         begin
           target.emit(tag, es, NullOutputChain.instance)
         rescue
-          $log.warn "failed to emit", :error=>$!.to_s, :pid=>Process.pid
+          $log.warn "failed to emit", error: $!.to_s, pid: Process.pid
           $log.warn_backtrace
         end
       }
     rescue
-      $log.error "error on output process forwarding thread", :error=>$!.to_s, :pid=>Process.pid
+      $log.error "error on output process forwarding thread", error: $!.to_s, pid: Process.pid
       $log.error_backtrace
       raise
     end
@@ -165,12 +171,12 @@ module Fluent
         begin
           Engine.emit_stream(tag, es)
         rescue
-          $log.warn "failed to emit", :error=>$!.to_s, :pid=>Process.pid
+          $log.warn "failed to emit", error: $!.to_s, pid: Process.pid
           $log.warn_backtrace
         end
       }
     rescue
-      $log.error "error on input process forwarding thread", :error=>$!.to_s, :pid=>Process.pid
+      $log.error "error on input process forwarding thread", error: $!.to_s, pid: Process.pid
       $log.error_backtrace
       raise
     end
@@ -264,7 +270,7 @@ module Fluent
           end
         end
       rescue
-        $log.error "error on forwerder thread", :error=>$!.to_s
+        $log.error "error on forwerder thread", error: $!.to_s
         $log.error_backtrace
         raise
       end
@@ -302,7 +308,7 @@ module Fluent
 
         if pid
           # parent process
-          $log.info "detached process", :class=>self.class, :pid=>pid
+          $log.info "detached process", class: self.class, pid: pid
           children << [pid, forward_thread]
           next
         end
@@ -332,7 +338,7 @@ module Fluent
           on_exit_process(i)
           exit! 0
         ensure
-          $log.error "unknown error while shutting down this child process", :error=>$!.to_s, :pid=>Process.pid
+          $log.error "unknown error while shutting down this child process", error: $!.to_s, pid: Process.pid
           $log.error_backtrace
         end
 
@@ -353,7 +359,7 @@ module Fluent
               pair[0] = nil
             end
           rescue
-            $log.error "unknown error while shutting down remote child process", :error=>$!.to_s
+            $log.error "unknown error while shutting down remote child process", error: $!.to_s
             $log.error_backtrace
           end
         }

data/lib/fluent/registry.rb

@@ -14,9 +14,9 @@
 #    limitations under the License.
 #
 
-module Fluent
-  require 'fluent/config/error'
+require 'fluent/config/error'
 
+module Fluent
   class Registry
     def initialize(kind, search_prefix)
       @kind = kind

data/lib/fluent/root_agent.rb

@@ -13,13 +13,17 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
-module Fluent
 
-  require 'delegate'
+require 'delegate'
 
-  require 'fluent/agent'
-  require 'fluent/label'
+require 'fluent/config/error'
+require 'fluent/agent'
+require 'fluent/engine'
+require 'fluent/label'
+require 'fluent/plugin'
+require 'fluent/system_config'
 
+module Fluent
   #
   # Fluentd forms a tree structure to manage plugins:
   #
@@ -43,7 +47,7 @@ module Fluent
   class RootAgent < Agent
     ERROR_LABEL = "@ERROR".freeze # @ERROR is built-in error label
 
-    def initialize(opts = {})
+    def initialize(system_config = SystemConfig.new)
       super
 
       @labels = {}
@@ -52,8 +56,8 @@ module Fluent
       @suppress_emit_error_log_interval = 0
       @next_emit_error_log_time = nil
 
-      suppress_interval(opts[:suppress_interval]) if opts[:suppress_interval]
-      @without_source = opts[:without_source] if opts[:without_source]
+      suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
+      @without_source = system_config.without_source unless system_config.without_source.nil?
     end
 
     attr_reader :inputs
@@ -121,7 +125,7 @@ module Fluent
             log.info "shutting down input", type: Plugin.lookup_name_from_class(i.class), plugin_id: i.plugin_id
             i.shutdown
           rescue => e
-            log.warn "unexpected error while shutting down input plugin", :plugin => i.class, :plugin_id => i.plugin_id, :error_class => e.class, :error => e
+            log.warn "unexpected error while shutting down input plugin", plugin: i.class, plugin_id: i.plugin_id, error_class: e.class, error: e
             log.warn_backtrace
           end
         end
@@ -168,7 +172,7 @@ module Fluent
     end
 
     def emit_error_event(tag, time, record, error)
-      error_info = {:error_class => error.class, :error => error.to_s, :tag => tag, :time => time}
+      error_info = {error_class: error.class, error: error.to_s, tag: tag, time: time}
       if @error_collector
         # A record is not included in the logs because <@ERROR> handles it. This warn is for the notification
         log.warn "send an error event to @ERROR:", error_info
@@ -180,7 +184,7 @@ module Fluent
     end
 
     def handle_emits_error(tag, es, error)
-      error_info = {:error_class => error.class, :error => error.to_s, :tag => tag}
+      error_info = {error_class: error.class, error: error.to_s, tag: tag}
       if @error_collector
         log.warn "send an error event stream to @ERROR:", error_info
         @error_collector.emit_stream(tag, es)
@@ -209,14 +213,14 @@ module Fluent
       end
 
       def emit_error_event(tag, time, record, error)
-        error_info = {:error_class => error.class, :error => error.to_s, :tag => tag, :time => time, :record => record}
+        error_info = {error_class: error.class, error: error.to_s, tag: tag, time: time, record: record}
         log.warn "dump an error event in @ERROR:", error_info
       end
 
       def handle_emits_error(tag, es, e)
         now = Engine.now
         if @suppress_emit_error_log_interval.zero? || now > @next_emit_error_log_time
-          log.warn "emit transaction failed in @ERROR:", :error_class => e.class, :error => e, :tag => tag
+          log.warn "emit transaction failed in @ERROR:", error_class: e.class, error: e, tag: tag
           log.warn_backtrace
           @next_emit_error_log_time = now + @suppress_emit_error_log_interval
         end

data/lib/fluent/rpc.rb

@@ -14,10 +14,10 @@
 #    limitations under the License.
 #
 
+require 'webrick'
+
 module Fluent
   module RPC
-    require 'webrick'
-
     class Server
       def initialize(endpoint, log)
         bind, port = endpoint.split(':')
@@ -26,10 +26,10 @@ module Fluent
         @log = log
 
         @server = WEBrick::HTTPServer.new(
-          :BindAddress => @bind,
-          :Port => @port,
-          :Logger => WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
-          :AccessLog => [],
+          BindAddress: @bind,
+          Port: @port,
+          Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
+          AccessLog: [],
         )
       end
 
@@ -43,7 +43,7 @@ module Fluent
           begin
             code, header, response = block.call(req, res)
           rescue => e
-            @log.warn "failed to handle RPC request", :path => path, :error => e.to_s
+            @log.warn "failed to handle RPC request", path: path, error: e.to_s
             @log.warn_backtrace e.backtrace
 
             code = 500

data/lib/fluent/supervisor.rb

@@ -14,9 +14,16 @@
 #    limitations under the License.
 #
 
-require 'fluent/load'
 require 'etc'
 
+require 'fluent/config'
+require 'fluent/env'
+require 'fluent/engine'
+require 'fluent/log'
+require 'fluent/plugin'
+require 'fluent/rpc'
+require 'fluent/system_config'
+
 module Fluent
   class Supervisor
     def self.get_etc_passwd(user)
@@ -80,20 +87,20 @@ module Fluent
 
     def self.default_options
       {
-        :config_path => Fluent::DEFAULT_CONFIG_PATH,
-        :plugin_dirs => [Fluent::DEFAULT_PLUGIN_DIR],
-        :log_level => Fluent::Log::LEVEL_INFO,
-        :log_path => nil,
-        :daemonize => nil,
-        :libs => [],
-        :setup_path => nil,
-        :chuser => nil,
-        :chgroup => nil,
-        :suppress_interval => 0,
-        :suppress_repeated_stacktrace => true,
-        :without_source => false,
-        :use_v1_config => true,
-        :supervise => true,
+        config_path: Fluent::DEFAULT_CONFIG_PATH,
+        plugin_dirs: [Fluent::DEFAULT_PLUGIN_DIR],
+        log_level: Fluent::Log::LEVEL_INFO,
+        log_path: nil,
+        daemonize: nil,
+        libs: [],
+        setup_path: nil,
+        chuser: nil,
+        chgroup: nil,
+        suppress_interval: 0,
+        suppress_repeated_stacktrace: true,
+        without_source: false,
+        use_v1_config: true,
+        supervise: true,
       }
     end
 
@@ -118,7 +125,7 @@ module Fluent
       @suppress_config_dump = opt[:suppress_config_dump]
       @without_source = opt[:without_source]
 
-      log_opts = {:suppress_repeated_stacktrace => opt[:suppress_repeated_stacktrace]}
+      log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace]}
       @log = LoggerInitializer.new(@log_path, @log_level, @chuser, @chgroup, log_opts)
       @finished = false
       @main_pid = nil
@@ -128,7 +135,7 @@ module Fluent
       @log.init
       show_plugin_config if @show_plugin_config
       read_config
-      apply_system_config
+      set_system_config
 
       dry_run if @dry_run
       start_daemonize if @daemonize
@@ -337,7 +344,7 @@ module Fluent
       @main_pid = nil
       ecode = $?.to_i
 
-      $log.info "process finished", :code=>ecode
+      $log.info "process finished", code: ecode
 
       if !@finished && Time.now - start_time < 1
         $log.warn "process died within 1 second. exit."
@@ -352,20 +359,20 @@ module Fluent
         block.call
 
       rescue Fluent::ConfigError
-        $log.error "config error", :file=>@config_path, :error=>$!.to_s
+        $log.error "config error", file: @config_path, error: $!.to_s
         $log.debug_backtrace
         unless @log.stdout?
           console = Fluent::Log.new(STDOUT, @log_level).enable_debug
-          console.error "config error", :file=>@config_path, :error=>$!.to_s
+          console.error "config error", file: @config_path, error: $!.to_s
           console.debug_backtrace
         end
 
       rescue
-        $log.error "unexpected error", :error=>$!.to_s
+        $log.error "unexpected error", error: $!.to_s
         $log.error_backtrace
         unless @log.stdout?
           console = Fluent::Log.new(STDOUT, @log_level).enable_debug
-          console.error "unexpected error", :error=>$!.to_s
+          console.error "unexpected error", error: $!.to_s
           console.error_backtrace
         end
       end
@@ -429,7 +436,7 @@ module Fluent
       # in main thread during trap context
       Thread.new {
         read_config
-        apply_system_config
+        set_system_config
         if pid = @main_pid
           Process.kill(:TERM, pid)
           # don't resuce Erro::ESRSH here (invalid status)
@@ -454,7 +461,7 @@ module Fluent
     end
 
     def read_config
-      $log.info "reading config file", :path => @config_path
+      $log.info "reading config file", path: @config_path
       @config_fname = File.basename(@config_path)
       @config_basedir = File.dirname(@config_path)
       @config_data = File.read(@config_path)
@@ -466,49 +473,9 @@ module Fluent
       @conf = Fluent::Config.parse(@config_data, @config_fname, @config_basedir, @use_v1_config)
     end
 
-    class SystemConfig
-      include Configurable
-
-      config_param :log_level, :default => nil do |level|
-        Log.str_to_level(level)
-      end
-      config_param :suppress_repeated_stacktrace, :bool, :default => nil
-      config_param :emit_error_log_interval, :time, :default => nil
-      config_param :suppress_config_dump, :bool, :default => nil
-      config_param :without_source, :bool, :default => nil
-      config_param :rpc_endpoint, :string, :default => nil
-      config_param :enable_get_dump, :bool, :default => nil
-      config_param :process_name, :default => nil
-
-      def initialize(conf)
-        super()
-        configure(conf)
-      end
-
-      def apply(supervisor)
-        system = self
-        supervisor.instance_eval {
-          @log.level = @log_level = system.log_level unless system.log_level.nil?
-          @suppress_interval = system.emit_error_log_interval unless system.emit_error_log_interval.nil?
-          @suppress_config_dump = system.suppress_config_dump unless system.suppress_config_dump.nil?
-          @suppress_repeated_stacktrace = system.suppress_repeated_stacktrace unless system.suppress_repeated_stacktrace.nil?
-          @without_source = system.without_source unless system.without_source.nil?
-          @rpc_endpoint = system.rpc_endpoint unless system.rpc_endpoint.nil?
-          @enable_get_dump = system.enable_get_dump unless system.enable_get_dump.nil?
-          @process_name = system.process_name unless system.process_name.nil?
-        }
-      end
-    end
-
-    # TODO: this method should be moved to SystemConfig class method
-    def apply_system_config
-      systems = @conf.elements.select { |e|
-        e.name == 'system'
-      }
-      return if systems.empty?
-      raise ConfigError, "<system> is duplicated. <system> should be only one" if systems.size > 1
-
-      SystemConfig.new(systems.first).apply(self)
+    def set_system_config
+      @system_config = SystemConfig.create(@conf) # @conf is set in read_config
+      @system_config.apply(self)
     end
 
     def run_configure
@@ -533,8 +500,7 @@ module Fluent
     end
 
     def init_engine
-      init_opts = {:suppress_interval => @suppress_interval, :suppress_config_dump => @suppress_config_dump, :without_source => @without_source}
-      Fluent::Engine.init(init_opts)
+      Fluent::Engine.init(@system_config)
 
       @libs.each {|lib|
         require lib

data/lib/fluent/system_config.rb

@@ -0,0 +1,98 @@
+#
+# Fluent
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/configurable'
+require 'fluent/config/element'
+
+module Fluent
+  module SystemConfigMixin
+    def system_config
+      @_system_config || Fluent::Engine.system_config
+    end
+
+    def system_config_override(opts={})
+      unless @_system_config
+        @_system_config = Fluent::Engine.system_config.dup
+      end
+      opts.each_pair do |key, value|
+        @_system_config.send(:"#{key.to_s}=", value)
+      end
+    end
+  end
+
+  class SystemConfig
+    include Configurable
+
+    config_param :log_level, default: nil do |level|
+      Log.str_to_level(level)
+    end
+    config_param :suppress_repeated_stacktrace, :bool, default: nil
+    config_param :emit_error_log_interval, :time, default: nil
+    config_param :suppress_config_dump, :bool, default: nil
+    config_param :without_source, :bool, default: nil
+    config_param :rpc_endpoint, :string, default: nil
+    config_param :enable_get_dump, :bool, default: nil
+    config_param :process_name, default: nil
+
+    def self.create(conf)
+      systems = conf.elements.select { |e|
+        e.name == 'system'
+      }
+      return SystemConfig.new if systems.empty?
+      raise Fluent::ConfigError, "<system> is duplicated. <system> should be only one" if systems.size > 1
+
+      SystemConfig.new(systems.first)
+    end
+
+    def self.blank_system_config
+      Fluent::Config::Element.new('<SYSTEM>', '', {}, [])
+    end
+
+    def initialize(conf=nil)
+      super()
+      conf ||= SystemConfig.blank_system_config
+      configure(conf)
+    end
+
+    def dup
+      s = SystemConfig.new
+      s.log_level = @log_level
+      s.suppress_repeated_stacktrace = @suppress_repeated_stacktrace
+      s.emit_error_log_interval = @emit_error_log_interval
+      s.suppress_config_dump = @suppress_config_dump
+      s.without_source = @without_source
+      s.rpc_endpoint = @rpc_endpoint
+      s.enable_get_dump = @enable_get_dump
+      s.process_name = @process_name
+
+      s
+    end
+
+    def apply(supervisor)
+      system = self
+      supervisor.instance_eval {
+        @log.level = @log_level = system.log_level unless system.log_level.nil?
+        @suppress_interval = system.emit_error_log_interval unless system.emit_error_log_interval.nil?
+        @suppress_config_dump = system.suppress_config_dump unless system.suppress_config_dump.nil?
+        @suppress_repeated_stacktrace = system.suppress_repeated_stacktrace unless system.suppress_repeated_stacktrace.nil?
+        @without_source = system.without_source unless system.without_source.nil?
+        @rpc_endpoint = system.rpc_endpoint unless system.rpc_endpoint.nil?
+        @enable_get_dump = system.enable_get_dump unless system.enable_get_dump.nil?
+        @process_name = system.process_name unless system.process_name.nil?
+      }
+    end
+  end
+end