fluentd 0.12.43 → 0.14.0

data/lib/fluent/plugin_helper/timer.rb

@@ -0,0 +1,85 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin_helper/event_loop'
+
+module Fluent
+  module PluginHelper
+    module Timer
+      include Fluent::PluginHelper::EventLoop
+
+      # stop     : turn checker into false (callbacks not called anymore)
+      # shutdown : [-]
+      # close    : [-]
+      # terminate: [-]
+
+      attr_reader :_timers # for tests
+
+      # interval: integer/float, repeat: true/false
+      def timer_execute(title, interval, repeat: true, &block)
+        raise ArgumentError, "BUG: title must be a symbol" unless title.is_a? Symbol
+        raise ArgumentError, "BUG: block not specified for callback" unless block_given?
+        checker = ->(){ @_timer_running }
+        timer = TimerWatcher.new(title, interval, repeat, log, checker, &block)
+        @_timers << title
+        event_loop_attach(timer)
+      end
+
+      def timer_running?
+        defined?(@_timer_running) && @_timer_running
+      end
+
+      def initialize
+        super
+        @_timers = []
+      end
+
+      def start
+        super
+        @_timer_running = true
+      end
+
+      def stop
+        super
+        @_timer_running = false
+      end
+
+      def terminate
+        super
+        @_timers = []
+      end
+
+      class TimerWatcher < Coolio::TimerWatcher
+        def initialize(title, interval, repeat, log, checker, &callback)
+          @title = title
+          @callback = callback
+          @log = log
+          @checker = checker
+          super(interval, repeat)
+        end
+
+        def on_timer
+          @callback.call if @checker.call
+        rescue => e
+          @log.error "Unexpected error raised. Stopping the timer.", title: @title, error: e
+          @log.error_backtrace
+          self.detach
+          @log.error "Timer detached.", title: @title
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_id.rb

@@ -0,0 +1,63 @@
+#
+# Fluent
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'set'
+
+module Fluent
+  module PluginId
+    @@configured_ids = Set.new
+
+    def configure(conf)
+      @id = conf['@id']
+      @_id_configured = !!@id # plugin id is explicitly configured by users (or not)
+      if @id
+        @id = @id.to_s
+        if @@configured_ids.include?(@id) && !plugin_id_for_test?
+          raise Fluent::ConfigError, "Duplicated plugin id `#{@id}`. Check whole configuration and fix it."
+        end
+        @@configured_ids.add(@id)
+      end
+
+      super
+    end
+
+    def plugin_id_for_test?
+      caller_locations.each do |location|
+        # Thread::Backtrace::Location#path returns base filename or absolute path.
+        # #absolute_path returns absolute_path always.
+        # https://bugs.ruby-lang.org/issues/12159
+        if location.absolute_path =~ /\/test_[^\/]+\.rb$/ # location.path =~ /test_.+\.rb$/
+          return true
+        end
+      end
+      false
+    end
+
+    def plugin_id_configured?
+      if instance_variable_defined?("@_id_configured")
+        @_id_configured
+      end
+    end
+
+    def plugin_id
+      if instance_variable_defined?("@id")
+        @id || "object:#{object_id.to_s(16)}"
+      else
+        "object:#{object_id.to_s(16)}"
+      end
+    end
+  end
+end

data/lib/fluent/process.rb

@@ -40,23 +40,23 @@ module Fluent
       @parent_uri = DRb.uri
     end
 
-    def fork(forward_interval, delegate_object)
+    def fork(delegate_object)
       ipr, ipw = IO.pipe  # child Engine.emit_stream -> parent Engine.emit_stream
-      opr, opw = IO.pipe  # parent target.emit -> child target.emit
+      opr, opw = IO.pipe  # parent target.emit_events -> child target.emit_events
 
       pid = Process.fork
       if pid
         # parent process
         ipw.close
         opr.close
-        forward_thread = process_parent(ipr, opw, pid, forward_interval, delegate_object)
+        forward_thread = process_parent(ipr, opw, pid, delegate_object)
         return pid, forward_thread
       end
 
       # child process
       ipr.close
       opw.close
-      forward_thread = process_child(ipw, opr, forward_interval, delegate_object)
+      forward_thread = process_child(ipw, opr, delegate_object)
       return nil, forward_thread
     end
 
@@ -77,14 +77,14 @@ module Fluent
     end
 
     private
-    def process_child(ipw, opr, forward_interval, delegate_object)
+    def process_child(ipw, opr, delegate_object)
       DRb.start_service(create_drb_uri, delegate_object)
       child_uri = DRb.uri
 
       send_header(ipw, child_uri)
 
       # override target.emit_stream to write event stream to the pipe
-      fwd = new_forwarder(ipw, forward_interval)
+      fwd = new_forwarder(ipw, 0.5)  # TODO interval
       Engine.define_singleton_method(:emit_stream) do |tag,es|
         fwd.emit(tag, es)
       end
@@ -115,8 +115,8 @@ module Fluent
       ]
     end
 
-    def process_parent(ipr, opw, pid, forward_interval, delegate_object)
-      child_uri = read_header(ipr)
+    def process_parent(ipr, opw, pid, delegate_object)
+      # child_uri = read_header(ipr)
 
       # read event stream from the pipe and forward to Engine.emit_stream
       forward_thread = Thread.new(ipr, pid, &method(:input_forward_main))
@@ -127,7 +127,7 @@ module Fluent
 
       # return forwarder for DetachProcessMixin to
       # override target.emit and write event stream to the pipe
-      fwd = new_forwarder(opw, forward_interval)
+      fwd = new_forwarder(opw, 0.5)  # TODO interval
       # note: override emit method on DetachProcessMixin
       forward_thread.define_singleton_method(:forwarder) do
         fwd
@@ -153,7 +153,7 @@ module Fluent
       read_event_stream(opr) {|tag,es|
         # FIXME error handling
         begin
-          target.emit(tag, es, NullOutputChain.instance)
+          target.emit_events(tag, es)
         rescue
           $log.warn "failed to emit", error: $!.to_s, pid: Process.pid
           $log.warn_backtrace
@@ -300,11 +300,11 @@ module Fluent
 
     private
 
-    def detach_process_impl(num, forward_interval, &block)
+    def detach_process_impl(num, &block)
       children = []
 
       num.times do |i|
-        pid, forward_thread = DetachProcessManager.instance.fork(forward_interval, self)
+        pid, forward_thread = DetachProcessManager.instance.fork(self)
 
         if pid
           # parent process
@@ -365,7 +365,7 @@ module Fluent
         }
       end
 
-      # override target.emit and write event stream to the pipe
+      # override target.emit_events and write event stream to the pipe
       forwarders = children.map {|pair| pair[1].forwarder }
       if forwarders.length > 1
         # use roundrobin
@@ -373,9 +373,8 @@ module Fluent
       else
         fwd = forwarders[0]
       end
-      define_singleton_method(:emit) do |tag,es,chain|
-        chain.next
-        fwd.emit(tag, es)
+      define_singleton_method(:emit_events) do |tag,es|
+        fwd.emit_events(tag, es)
       end
     end
 
@@ -418,6 +417,8 @@ module Fluent
     def configure(conf)
       super
 
+      @detach_process = nil
+
       if detach_process = conf['detach_process']
         b3v = Config.bool_value(detach_process)
         case b3v
@@ -438,17 +439,11 @@ module Fluent
           @detach_process = false
         end
       end
-
-      if forward_interval = conf['detach_process_forward_interval']
-        @forward_interval = Config.time_value(forward_interval)
-      else
-        @forward_interval = 0.5
-      end
     end
 
     def detach_process(&block)
       if @detach_process
-        detach_process_impl(1, @forward_interval, &block)
+        detach_process_impl(1, &block)
       else
         block.call
       end
@@ -467,6 +462,8 @@ module Fluent
     def configure(conf)
       super
 
+      @detach_process = nil
+
       if detach_process = conf['detach_process']
         b3v = Config.bool_value(detach_process)
         case b3v
@@ -486,19 +483,13 @@ module Fluent
           @detach_process = false
         end
       end
-
-      if forward_interval = conf['detach_process_forward_interval']
-        @forward_interval = Config.time_value(forward_interval)
-      else
-        @forward_interval = 0.5
-      end
     end
 
     private
 
     def detach_multi_process(&block)
       if @detach_process
-        detach_process_impl(@detach_process_num, @forward_interval, &block)
+        detach_process_impl(@detach_process_num, &block)
       else
         block.call
       end

data/lib/fluent/registry.rb

@@ -18,13 +18,16 @@ require 'fluent/config/error'
 
 module Fluent
   class Registry
+    DEFAULT_PLUGIN_PATH = File.expand_path('plugin', __FILE__)
+
     def initialize(kind, search_prefix)
       @kind = kind
       @search_prefix = search_prefix
       @map = {}
+      @paths = [DEFAULT_PLUGIN_PATH]
     end
 
-    attr_reader :kind
+    attr_reader :kind, :paths
 
     def register(type, value)
       type = type.to_sym
@@ -43,18 +46,27 @@ module Fluent
       raise ConfigError, "Unknown #{@kind} plugin '#{type}'. Run 'gem search -rd fluent-plugin' to find plugins"  # TODO error class
     end
 
+    def reverse_lookup(value)
+      @map.each do |k, v|
+        return k if v == value
+      end
+      nil
+    end
+
     def search(type)
       path = "#{@search_prefix}#{type}"
 
       # prefer LOAD_PATH than gems
-      files = $LOAD_PATH.map { |lp|
-        lpath = File.expand_path(File.join(lp, "#{path}.rb"))
-        File.exist?(lpath) ? lpath : nil
-      }.compact
-      unless files.empty?
-        # prefer newer version
-        require files.sort.last
-        return
+      [@paths, $LOAD_PATH].each do |paths|
+        files = paths.map { |lp|
+          lpath = File.expand_path(File.join(lp, "#{path}.rb"))
+          File.exist?(lpath) ? lpath : nil
+        }.compact
+        unless files.empty?
+          # prefer newer version
+          require files.sort.last
+          return
+        end
       end
 
       specs = Gem::Specification.find_all { |spec|

data/lib/fluent/root_agent.rb

@@ -18,10 +18,10 @@ require 'delegate'
 
 require 'fluent/config/error'
 require 'fluent/agent'
-require 'fluent/engine'
 require 'fluent/label'
 require 'fluent/plugin'
 require 'fluent/system_config'
+require 'fluent/time'
 
 module Fluent
   #
@@ -47,14 +47,14 @@ module Fluent
   class RootAgent < Agent
     ERROR_LABEL = "@ERROR".freeze # @ERROR is built-in error label
 
-    def initialize(system_config = SystemConfig.new)
-      super
+    def initialize(log:, system_config: SystemConfig.new)
+      super(log: log)
 
       @labels = {}
       @inputs = []
-      @started_inputs = []
       @suppress_emit_error_log_interval = 0
       @next_emit_error_log_time = nil
+      @without_source = false
 
       suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
       @without_source = system_config.without_source unless system_config.without_source.nil?
@@ -68,7 +68,7 @@ module Fluent
 
       # initialize <label> elements before configuring all plugins to avoid 'label not found' in input, filter and output.
       label_configs = {}
-      conf.elements.select { |e| e.name == 'label' }.each { |e|
+      conf.elements(name: 'label').each { |e|
         name = e.arg
         raise ConfigError, "Missing symbol argument on <label> directive" if name.empty?
 
@@ -89,9 +89,9 @@ module Fluent
       if @without_source
         log.info "'--without-source' is applied. Ignore <source> sections"
       else
-        conf.elements.select { |e| e.name == 'source' }.each { |e|
-          type = e['@type'] || e['type']
-          raise ConfigError, "Missing '@type' parameter on <source> directive" unless type
+        conf.elements(name: 'source').each { |e|
+          type = e['@type']
+          raise ConfigError, "Missing 'type' parameter on <source> directive" unless type
           add_source(type, e)
         }
       end
@@ -104,43 +104,118 @@ module Fluent
       @error_collector = error_label.event_router
     end
 
-    def start
-      super
+    def lifecycle(desc: false, kind_callback: nil)
+      kind_or_label_list = if desc
+                    [:output, :filter, @labels.values.reverse, :output_with_router, :input].flatten
+                  else
+                    [:input, :output_with_router, @labels.values, :filter, :output].flatten
+                  end
+      kind_or_label_list.each do |kind|
+        if kind.respond_to?(:lifecycle)
+          label = kind
+          label.lifecycle(desc: desc) do |plugin, display_kind|
+            yield plugin, display_kind
+          end
+        else
+          list = if desc
+                   lifecycle_control_list[kind].reverse
+                 else
+                   lifecycle_control_list[kind]
+                 end
+          display_kind = (kind == :output_with_router ? :output : kind)
+          list.each do |instance|
+            yield instance, display_kind
+          end
+        end
+        if kind_callback
+          kind_callback.call
+        end
+      end
+    end
 
-      @labels.each { |n, l|
-        l.start
-      }
+    def start
+      lifecycle(desc: true) do |i| # instance
+        i.start unless i.started?
+      end
+    end
 
-      @inputs.each { |i|
-        i.start
-        @started_inputs << i
-      }
+    def flush!
+      log.info "flushing all buffer forcedly"
+      flushing_threads = []
+      lifecycle(desc: true) do |instance|
+        if instance.respond_to?(:force_flush)
+          t = Thread.new do
+            Thread.current.abort_on_exception = true
+            begin
+              instance.force_flush
+            rescue => e
+              log.warn "unexpected error while flushing buffer", plugin: instance.class, plugin_id: instance.plugin_id, error: e
+              log.warn_backtrace
+            end
+          end
+          flushing_threads << t
+        end
+      end
+      flushing_threads.each{|t| t.join }
     end
 
-    def shutdown
-      # Shutdown Input plugin first to prevent emitting to terminated Output plugin
-      @started_inputs.map { |i|
-        Thread.new do
+    def shutdown # Fluentd's shutdown sequence is stop, before_shutdown, shutdown, after_shutdown, close, terminate for plugins
+      # Thesee method callers does `rescue Exception` to call methods of shutdown sequence as far as possible
+      # if plugin methods does something like infinite recursive call, `exit`, unregistering signal handlers or others.
+      # Plugins should be separated and be in sandbox to protect data in each plugins/buffers.
+
+      lifecycle_safe_sequence = ->(method, checker) {
+        lifecycle do |instance, kind|
           begin
-            log.info "shutting down input", type: Plugin.lookup_name_from_class(i.class), plugin_id: i.plugin_id
-            i.shutdown
-          rescue => e
-            log.warn "unexpected error while shutting down input plugin", plugin: i.class, plugin_id: i.plugin_id, error_class: e.class, error: e
+            log.debug "calling #{method} on #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
+            instance.send(method) unless instance.send(checker)
+          rescue Exception => e
+            log.warn "unexpected error while calling #{method} on #{kind} plugin", pluguin: instance.class, plugin_id: instance.plugin_id, error: e
             log.warn_backtrace
           end
         end
-      }.each { |t| t.join }
+      }
 
-      @labels.each { |n, l|
-        l.shutdown
+      lifecycle_unsafe_sequence = ->(method, checker) {
+        operation = case method
+                    when :before_shutdown then "preparing shutdown"
+                    when :shutdown then "shutting down"
+                    when :close    then "closing"
+                    else
+                      raise "BUG: unknown method name '#{method}'"
+                    end
+        operation_threads = []
+        callback = ->(){
+          operation_threads.each{|t| t.join }
+          operation_threads.clear
+        }
+        lifecycle(kind_callback: callback) do |instance, kind|
+          t = Thread.new do
+            Thread.current.abort_on_exception = true
+            begin
+              log.info "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
+              instance.send(method) unless instance.send(checker)
+            rescue Exception => e
+              log.warn "unexpected error while #{operation} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e
+              log.warn_backtrace
+            end
+          end
+          operation_threads << t
+        end
       }
 
-      super
-    end
+      lifecycle_safe_sequence.call(:stop, :stopped?)
 
-    def flush!
-      super
-      @labels.each{ |name, label| label.flush! }
+      # before_shutdown does force_flush for output plugins: it should block, so it's unsafe operation
+      lifecycle_unsafe_sequence.call(:before_shutdown, :before_shutdown?)
+
+      lifecycle_unsafe_sequence.call(:shutdown, :shutdown?)
+
+      lifecycle_safe_sequence.call(:after_shutdown, :after_shutdown?)
+
+      lifecycle_unsafe_sequence.call(:close, :closed?)
+
+      lifecycle_safe_sequence.call(:terminate, :terminated?)
     end
 
     def suppress_interval(interval_time)
@@ -163,7 +238,7 @@ module Fluent
     end
 
     def add_label(name)
-      label = Label.new(name)
+      label = Label.new(name, log: log)
       label.root_agent = self
       @labels[name] = label
     end
@@ -177,7 +252,7 @@ module Fluent
     end
 
     def emit_error_event(tag, time, record, error)
-      error_info = {error_class: error.class, error: error.to_s, location: (error.backtrace ? error.backtrace.first : nil), tag: tag, time: time}
+      error_info = {error: error, tag: tag, time: time}
       if @error_collector
         # A record is not included in the logs because <@ERROR> handles it. This warn is for the notification
         log.warn "send an error event to @ERROR:", error_info
@@ -189,12 +264,12 @@ module Fluent
     end
 
     def handle_emits_error(tag, es, error)
-      error_info = {error_class: error.class, error: error.to_s, location: (error.backtrace ? error.backtrace.first : nil), tag: tag}
+      error_info = {error: error, tag: tag}
       if @error_collector
         log.warn "send an error event stream to @ERROR:", error_info
         @error_collector.emit_stream(tag, es)
       else
-        now = Engine.now
+        now = Time.now
         if @suppress_emit_error_log_interval.zero? || now > @next_emit_error_log_time
           log.warn "emit transaction failed:", error_info
           log.warn_backtrace
@@ -218,14 +293,14 @@ module Fluent
       end
 
       def emit_error_event(tag, time, record, error)
-        error_info = {error_class: error.class, error: error.to_s, location: (error.backtrace ? error.backtrace.first : nil), tag: tag, time: time, record: record}
+        error_info = {error: error, tag: tag, time: time, record: record}
         log.warn "dump an error event in @ERROR:", error_info
       end
 
       def handle_emits_error(tag, es, e)
-        now = Engine.now
+        now = EventTime.now
         if @suppress_emit_error_log_interval.zero? || now > @next_emit_error_log_time
-          log.warn "emit transaction failed in @ERROR:", error_class: e.class, error: e, tag: tag
+          log.warn "emit transaction failed in @ERROR:", error: e, tag: tag
           log.warn_backtrace
           @next_emit_error_log_time = now + @suppress_emit_error_log_interval
         end