fluentd 0.12.40 → 1.6.2

data/lib/fluent/root_agent.rb

@@ -18,10 +18,10 @@ require 'delegate'
 
 require 'fluent/config/error'
 require 'fluent/agent'
-require 'fluent/engine'
 require 'fluent/label'
 require 'fluent/plugin'
 require 'fluent/system_config'
+require 'fluent/time'
 
 module Fluent
   #
@@ -47,14 +47,14 @@ module Fluent
   class RootAgent < Agent
     ERROR_LABEL = "@ERROR".freeze # @ERROR is built-in error label
 
-    def initialize(system_config = SystemConfig.new)
-      super
+    def initialize(log:, system_config: SystemConfig.new)
+      super(log: log)
 
       @labels = {}
       @inputs = []
-      @started_inputs = []
       @suppress_emit_error_log_interval = 0
       @next_emit_error_log_time = nil
+      @without_source = false
 
       suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
       @without_source = system_config.without_source unless system_config.without_source.nil?
@@ -64,11 +64,75 @@ module Fluent
     attr_reader :labels
 
     def configure(conf)
+      used_worker_ids = []
+      available_worker_ids = (0..Fluent::Engine.system_config.workers - 1).to_a
+      # initialize <worker> elements
+      conf.elements(name: 'worker').each do |e|
+        target_worker_id_str = e.arg
+        if target_worker_id_str.empty?
+          raise Fluent::ConfigError, "Missing worker id on <worker> directive"
+        end
+
+        target_worker_ids = target_worker_id_str.split("-")
+        if target_worker_ids.size == 2
+          first_worker_id = target_worker_ids.first.to_i
+          last_worker_id = target_worker_ids.last.to_i
+          if first_worker_id > last_worker_id
+            raise Fluent::ConfigError, "greater first_worker_id<#{first_worker_id}> than last_worker_id<#{last_worker_id}> specified by <worker> directive is not allowed. Available multi worker assign syntax is <smaller_worker_id>-<greater_worker_id>"
+          end
+          target_worker_ids = []
+          first_worker_id.step(last_worker_id, 1) do |worker_id|
+            target_worker_id = worker_id.to_i
+            target_worker_ids << target_worker_id
+
+            if target_worker_id < 0 || target_worker_id > (Fluent::Engine.system_config.workers - 1)
+              raise Fluent::ConfigError, "worker id #{target_worker_id} specified by <worker> directive is not allowed. Available worker id is between 0 and #{(Fluent::Engine.system_config.workers - 1)}"
+            end
+            available_worker_ids.delete(target_worker_id) if available_worker_ids.include?(target_worker_id)
+            if used_worker_ids.include?(target_worker_id) && !Fluent::Engine.dry_run_mode
+              raise Fluent::ConfigError, "specified worker_id<#{worker_id}> collisions is detected on <worker> directive. Available worker id(s): #{available_worker_ids}"
+            end
+            used_worker_ids << target_worker_id
+
+            e.elements.each do |elem|
+              unless ['source', 'match', 'filter', 'label'].include?(elem.name)
+                raise Fluent::ConfigError, "<worker> section cannot have <#{elem.name}> directive"
+              end
+            end
+
+            # On dry_run mode, all worker sections have to be configured on supervisor (recognized as worker_id = 0).
+            target_worker_ids = [0] if Fluent::Engine.dry_run_mode
+
+            unless target_worker_ids.empty?
+              e.set_target_worker_ids(target_worker_ids.uniq)
+            end
+          end
+        else
+          target_worker_id = target_worker_id_str.to_i
+          if target_worker_id < 0 || target_worker_id > (Fluent::Engine.system_config.workers - 1)
+            raise Fluent::ConfigError, "worker id #{target_worker_id} specified by <worker> directive is not allowed. Available worker id is between 0 and #{(Fluent::Engine.system_config.workers - 1)}"
+          end
+
+          ## On dry_run mode, all worker sections have to be configured on supervisor (recognized as worker_id = 0).
+          target_worker_id = 0 if Fluent::Engine.dry_run_mode
+
+          e.elements.each do |elem|
+            unless ['source', 'match', 'filter', 'label'].include?(elem.name)
+              raise Fluent::ConfigError, "<worker> section cannot have <#{elem.name}> directive"
+            end
+            elem.set_target_worker_id(target_worker_id)
+          end
+        end
+        conf += e
+      end
+      conf.elements.delete_if{|e| e.name == 'worker'}
+
       error_label_config = nil
 
       # initialize <label> elements before configuring all plugins to avoid 'label not found' in input, filter and output.
       label_configs = {}
-      conf.elements.select { |e| e.name == 'label' }.each { |e|
+      conf.elements(name: 'label').each { |e|
+        next if e.for_another_worker?
         name = e.arg
         raise ConfigError, "Missing symbol argument on <label> directive" if name.empty?
 
@@ -87,10 +151,11 @@ module Fluent
 
       # initialize <source> elements
       if @without_source
-        log.info "'--without-source' is applied. Ignore <source> sections"
+        log.info :worker0, "'--without-source' is applied. Ignore <source> sections"
       else
-        conf.elements.select { |e| e.name == 'source' }.each { |e|
-          type = e['@type'] || e['type']
+        conf.elements(name: 'source').each { |e|
+          next if e.for_another_worker?
+          type = e['@type']
           raise ConfigError, "Missing '@type' parameter on <source> directive" unless type
           add_source(type, e)
         }
@@ -104,43 +169,138 @@ module Fluent
       @error_collector = error_label.event_router
     end
 
-    def start
-      super
+    def lifecycle(desc: false, kind_callback: nil)
+      kind_or_label_list = if desc
+                    [:output, :filter, @labels.values.reverse, :output_with_router, :input].flatten
+                  else
+                    [:input, :output_with_router, @labels.values, :filter, :output].flatten
+                  end
+      kind_or_label_list.each do |kind|
+        if kind.respond_to?(:lifecycle)
+          label = kind
+          label.lifecycle(desc: desc) do |plugin, display_kind|
+            yield plugin, display_kind
+          end
+        else
+          list = if desc
+                   lifecycle_control_list[kind].reverse
+                 else
+                   lifecycle_control_list[kind]
+                 end
+          display_kind = (kind == :output_with_router ? :output : kind)
+          list.each do |instance|
+            yield instance, display_kind
+          end
+        end
+        if kind_callback
+          kind_callback.call
+        end
+      end
+    end
 
-      @labels.each { |n, l|
-        l.start
-      }
+    def start
+      lifecycle(desc: true) do |i| # instance
+        i.start unless i.started?
+        # Input#start sometimes emits lots of evetns with in_tail/`read_from_head true` case
+        # and it causes deadlock for small buffer/queue output. To avoid such problem,
+        # buffer related output threads should be run before `Input#start`.
+        # This is why after_start should be called immediately after start call.
+        # This depends on `desc: true` because calling plugin order of `desc: true` is
+        # Output, Filter, Label, Output with Router, then Input.
+        i.after_start unless i.after_started?
+      end
+    end
 
-      @inputs.each { |i|
-        i.start
-        @started_inputs << i
-      }
+    def flush!
+      log.info "flushing all buffer forcedly"
+      flushing_threads = []
+      lifecycle(desc: true) do |instance|
+        if instance.respond_to?(:force_flush)
+          t = Thread.new do
+            Thread.current.abort_on_exception = true
+            begin
+              instance.force_flush
+            rescue => e
+              log.warn "unexpected error while flushing buffer", plugin: instance.class, plugin_id: instance.plugin_id, error: e
+              log.warn_backtrace
+            end
+          end
+          flushing_threads << t
+        end
+      end
+      flushing_threads.each{|t| t.join }
     end
 
-    def shutdown
-      # Shutdown Input plugin first to prevent emitting to terminated Output plugin
-      @started_inputs.map { |i|
-        Thread.new do
+    def shutdown # Fluentd's shutdown sequence is stop, before_shutdown, shutdown, after_shutdown, close, terminate for plugins
+      # These method callers does `rescue Exception` to call methods of shutdown sequence as far as possible
+      # if plugin methods does something like infinite recursive call, `exit`, unregistering signal handlers or others.
+      # Plugins should be separated and be in sandbox to protect data in each plugins/buffers.
+
+      lifecycle_safe_sequence = ->(method, checker) {
+        lifecycle do |instance, kind|
           begin
-            log.info "shutting down input", type: Plugin.lookup_name_from_class(i.class), plugin_id: i.plugin_id
-            i.shutdown
-          rescue => e
-            log.warn "unexpected error while shutting down input plugin", plugin: i.class, plugin_id: i.plugin_id, error_class: e.class, error: e
+            log.debug "calling #{method} on #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
+            instance.send(method) unless instance.send(checker)
+          rescue Exception => e
+            log.warn "unexpected error while calling #{method} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e
             log.warn_backtrace
           end
         end
-      }.each { |t| t.join }
+      }
 
-      @labels.each { |n, l|
-        l.shutdown
+      lifecycle_unsafe_sequence = ->(method, checker) {
+        operation = case method
+                    when :shutdown then "shutting down"
+                    when :close    then "closing"
+                    else
+                      raise "BUG: unknown method name '#{method}'"
+                    end
+        operation_threads = []
+        callback = ->(){
+          operation_threads.each{|t| t.join }
+          operation_threads.clear
+        }
+        lifecycle(kind_callback: callback) do |instance, kind|
+          t = Thread.new do
+            Thread.current.abort_on_exception = true
+            begin
+              if method == :shutdown
+                # To avoid Input#shutdown and Output#before_shutdown mismatch problem, combine before_shutdown and shutdown call in one sequence.
+                # The problem is in_tail flushes buffered multiline in shutdown but output's flush_at_shutdown is invoked in before_shutdown
+                operation = "preparing shutdown" # for logging
+                log.debug "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
+                begin
+                  instance.send(:before_shutdown) unless instance.send(:before_shutdown?)
+                rescue Exception => e
+                  log.warn "unexpected error while #{operation} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e
+                  log.warn_backtrace
+                end
+                operation = "shutting down"
+                log.info "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
+                instance.send(:shutdown) unless instance.send(:shutdown?)
+              else
+                log.debug "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
+                instance.send(method) unless instance.send(checker)
+              end
+            rescue Exception => e
+              log.warn "unexpected error while #{operation} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e
+              log.warn_backtrace
+            end
+          end
+          operation_threads << t
+        end
       }
 
-      super
-    end
+      lifecycle_safe_sequence.call(:stop, :stopped?)
 
-    def flush!
-      super
-      @labels.each{ |name, label| label.flush! }
+      # before_shutdown does force_flush for output plugins: it should block, so it's unsafe operation
+      lifecycle_unsafe_sequence.call(:shutdown, :shutdown?)
+
+      lifecycle_safe_sequence.call(:after_shutdown, :after_shutdown?)
+
+      lifecycle_unsafe_sequence.call(:close, :closed?)
+
+      lifecycle_safe_sequence.call(:terminate, :terminated?)
     end
 
     def suppress_interval(interval_time)
@@ -149,13 +309,14 @@ module Fluent
     end
 
     def add_source(type, conf)
-      log.info "adding source", type: type
+      log_type = conf.for_this_worker? ? :default : :worker0
+      log.info log_type, "adding source", type: type
 
       input = Plugin.new_input(type)
       # <source> emits events to the top-level event router (RootAgent#event_router).
       # Input#configure overwrites event_router to a label's event_router if it has `@label` parameter.
       # See also 'fluentd/plugin/input.rb'
-      input.router = @event_router
+      input.context_router = @event_router
       input.configure(conf)
       @inputs << input
 
@@ -163,7 +324,8 @@ module Fluent
     end
 
     def add_label(name)
-      label = Label.new(name)
+      label = Label.new(name, log: log)
+      raise ConfigError, "Section <label #{name}> appears twice" if @labels[name]
       label.root_agent = self
       @labels[name] = label
     end
@@ -177,7 +339,7 @@ module Fluent
     end
 
     def emit_error_event(tag, time, record, error)
-      error_info = {error_class: error.class, error: error.to_s, tag: tag, time: time}
+      error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag, time: time}
       if @error_collector
         # A record is not included in the logs because <@ERROR> handles it. This warn is for the notification
         log.warn "send an error event to @ERROR:", error_info
@@ -189,12 +351,12 @@ module Fluent
     end
 
     def handle_emits_error(tag, es, error)
-      error_info = {error_class: error.class, error: error.to_s, tag: tag}
+      error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag}
       if @error_collector
         log.warn "send an error event stream to @ERROR:", error_info
         @error_collector.emit_stream(tag, es)
       else
-        now = Engine.now
+        now = Time.now.to_i
         if @suppress_emit_error_log_interval.zero? || now > @next_emit_error_log_time
           log.warn "emit transaction failed:", error_info
           log.warn_backtrace
@@ -218,14 +380,14 @@ module Fluent
       end
 
       def emit_error_event(tag, time, record, error)
-        error_info = {error_class: error.class, error: error.to_s, tag: tag, time: time, record: record}
+        error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag, time: time, record: record}
         log.warn "dump an error event in @ERROR:", error_info
       end
 
       def handle_emits_error(tag, es, e)
-        now = Engine.now
+        now = EventTime.now.to_i
         if @suppress_emit_error_log_interval.zero? || now > @next_emit_error_log_time
-          log.warn "emit transaction failed in @ERROR:", error_class: e.class, error: e, tag: tag
+          log.warn "emit transaction failed in @ERROR:", error: e, tag: tag
           log.warn_backtrace
           @next_emit_error_log_time = now + @suppress_emit_error_log_interval
         end