fluentd 0.14.2 → 0.14.5

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -20,13 +20,16 @@ require 'cgi'
 
 require 'cool.io'
 
-require 'fluent/input'
-require 'fluent/output'
-require 'fluent/filter'
+require 'fluent/plugin/input'
+require 'fluent/plugin/output'
+require 'fluent/plugin/multi_output'
+require 'fluent/plugin/filter'
 
-module Fluent
+module Fluent::Plugin
   class MonitorAgentInput < Input
-    Plugin.register_input('monitor_agent', self)
+    Fluent::Plugin.register_input('monitor_agent', self)
+
+    helpers :timer, :thread
 
     config_param :bind, :string, default: '0.0.0.0'
     config_param :port, :integer, default: 24220
@@ -142,7 +145,7 @@ module Fluent
 
     class LTSVMonitorServlet < MonitorServlet
       def process(req, res)
-        list, opts = build_object(req, res)
+        list, _opts = build_object(req, res)
         return unless list
 
         normalized = JSON.parse(list.to_json)
@@ -204,36 +207,6 @@ module Fluent
       end
     end
 
-    class TimerWatcher < Coolio::TimerWatcher
-      def initialize(interval, log, &callback)
-        @callback = callback
-        @log = log
-
-        # Avoid long shutdown time
-        @num_call = 0
-        if interval >= 10
-          min_interval = 10
-          @call_interval = interval / 10
-        else
-          min_interval = interval
-          @call_interval = 0
-        end
-
-        super(min_interval, true)
-      end
-
-      def on_timer
-        @num_call += 1
-        if @num_call >= @call_interval
-          @num_call = 0
-          @callback.call
-        end
-      rescue => e
-        @log.error e.to_s
-        @log.error_backtrace
-      end
-    end
-
     def start
       super
 
@@ -248,76 +221,55 @@ module Fluent
       @srv.mount('/api/plugins.json', JSONMonitorServlet, self)
       @srv.mount('/api/config', LTSVConfigMonitorServlet, self)
       @srv.mount('/api/config.json', JSONConfigMonitorServlet, self)
-      @thread = Thread.new {
+      thread_create :in_monitor_agent_servlet do
         @srv.start
-      }
+      end
       if @tag
         log.debug "tag parameter is specified. Emit plugins info to '#{@tag}'"
 
-        @loop = Coolio::Loop.new
         opts = {with_config: false}
-        timer = TimerWatcher.new(@emit_interval, log) {
-          es = MultiEventStream.new
-          now = Engine.now
+        timer_execute(:in_monitor_agent_emit, @emit_interval, repeat: true) {
+          es = Fluent::MultiEventStream.new
+          now = Fluent::Engine.now
           plugins_info_all(opts).each { |record|
             es.add(now, record)
           }
           router.emit_stream(@tag, es)
         }
-        @loop.attach(timer)
-        @thread_for_emit = Thread.new(&method(:run))
       end
     end
 
-    def run
-      @loop.run
-    rescue => e
-      log.error "unexpected error", error: e.to_s
-      log.error_backtrace
-    end
-
     def shutdown
       if @srv
         @srv.shutdown
         @srv = nil
       end
-      if @thread
-        @thread.join
-        @thread = nil
-      end
-      if @tag
-        @loop.watchers.each { |w| w.detach }
-        @loop.stop
-        @loop = nil
-        @thread_for_emit.join
-        @thread_for_emit = nil
-      end
 
       super
     end
 
     MONITOR_INFO = {
-      'output_plugin' => 'is_a?(::Fluent::Plugin::Output)',
-      'buffer_queue_length' => '@buffer.queue.size',
-      'buffer_total_queued_size' => '@buffer.stage_size + @buffer.queue_size',
-      'retry_count' => '@num_errors',
+      'output_plugin' => ->(){ is_a?(::Fluent::Plugin::Output) },
+      'buffer_queue_length' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer); @buffer.queue.size },
+      'buffer_total_queued_size' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer); @buffer.stage_size },
+      'retry_count' => ->(){ instance_variable_defined?(:@num_errors) ? @num_errors : nil },
     }
 
     def all_plugins
       array = []
 
       # get all input plugins
-      array.concat Engine.root_agent.inputs
+      array.concat Fluent::Engine.root_agent.inputs
 
       # get all output plugins
-      Engine.root_agent.outputs.each { |o|
+      Fluent::Engine.root_agent.outputs.each { |o|
         MonitorAgentInput.collect_children(o, array)
       }
       # get all filter plugins
-      Engine.root_agent.filters.each { |f|
+      Fluent::Engine.root_agent.filters.each { |f|
         MonitorAgentInput.collect_children(f, array)
       }
-      Engine.root_agent.labels.each { |name, l|
+      Fluent::Engine.root_agent.labels.each { |name, l|
         # TODO: Add label name to outputs / filters for identifing plugins
         l.outputs.each { |o| MonitorAgentInput.collect_children(o, array) }
         l.filters.each { |f| MonitorAgentInput.collect_children(f, array) }
@@ -330,7 +282,7 @@ module Fluent
     # from the plugin `pe` recursively
     def self.collect_children(pe, array=[])
       array << pe
-      if pe.is_a?(MultiOutput) && pe.respond_to?(:outputs)
+      if pe.is_a?(Fluent::Plugin::MultiOutput) || pe.is_a?(Fluent::MultiOutput) && pe.respond_to?(:outputs)
         pe.outputs.each {|nop|
           collect_children(nop, array)
         }
@@ -341,10 +293,10 @@ module Fluent
     # try to match the tag and get the info from the matched output plugin
     # TODO: Support output in label
     def plugin_info_by_tag(tag, opts={})
-      matches = Engine.root_agent.event_router.instance_variable_get(:@match_rules)
+      matches = Fluent::Engine.root_agent.event_router.instance_variable_get(:@match_rules)
       matches.each { |rule|
         if rule.match?(tag)
-          if rule.collector.is_a?(Output)
+          if rule.collector.is_a?(Fluent::Plugin::Output) || rule.collector.is_a?(Fluent::Output)
             return get_monitor_info(rule.collector, opts)
           end
         end
@@ -381,8 +333,7 @@ module Fluent
       }
     end
 
-    # TODO: use %i() after drop ruby v1.9.3 support.
-    IGNORE_ATTRIBUTES = %W(@config_root_section @config @masked_config).map(&:to_sym)
+    IGNORE_ATTRIBUTES = %i(@config_root_section @config @masked_config)
 
     # get monitor info from the plugin `pe` and return a hash object
     def get_monitor_info(pe, opts={})
@@ -397,8 +348,11 @@ module Fluent
       # run MONITOR_INFO in plugins' instance context and store the info to obj
       MONITOR_INFO.each_pair {|key,code|
         begin
-          obj[key] = pe.instance_eval(code)
-        rescue
+          catch(:skip) do
+            obj[key] = pe.instance_exec(&code)
+          end
+        rescue => e
+          log.warn "unexpected error in monitoring plugins", key: key, plugin: pe.class, error: e
         end
       }
 

data/lib/fluent/plugin/in_tail.rb

@@ -16,10 +16,9 @@
 
 require 'cool.io'
 
-require 'fluent/input'
+require 'fluent/plugin/input'
 require 'fluent/config/error'
 require 'fluent/event'
-require 'fluent/system_config'
 require 'fluent/plugin/buffer'
 
 if Fluent.windows?
@@ -28,11 +27,11 @@ else
   Fluent::FileWrapper = File
 end
 
-module Fluent
-  class TailInput < Input
-    include SystemConfig::Mixin
+module Fluent::Plugin
+  class TailInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('tail', self)
 
-    Plugin.register_input('tail', self)
+    helpers :timer, :event_loop, :parser, :compat_parameters
 
     FILE_PERMISSION = 0644
 
@@ -67,25 +66,34 @@ module Fluent
     config_param :multiline_flush_interval, :time, default: nil
     desc 'Enable the additional watch timer.'
     config_param :enable_watch_timer, :bool, default: true
+    desc 'The encoding after conversion of the input.'
+    config_param :encoding, :string, default: nil
     desc 'The encoding of the input.'
-    config_param :encoding, default: nil do |encoding_name|
-      begin
-        Encoding.find(encoding_name)
-      rescue ArgumentError => e
-        raise ConfigError, e.message
-      end
-    end
+    config_param :from_encoding, :string, default: nil
     desc 'Add the log path being tailed to records. Specify the field name to be used.'
     config_param :path_key, :string, default: nil
 
     attr_reader :paths
 
     def configure(conf)
+      compat_parameters_convert(conf, :parser)
+      parser_config = conf.elements('parse').first
+      unless parser_config
+        raise Fluent::ConfigError, "<parse> section is required."
+      end
+      unless parser_config["@type"]
+        raise Fluent::ConfigError, "parse/@type is required."
+      end
+
+      (1..Fluent::Plugin::MultilineParser::FORMAT_MAX_NUM).each do |n|
+        parser_config["format#{n}"] = conf["format#{n}"] if conf["format#{n}"]
+      end
+
       super
 
       @paths = @path.split(',').map {|path| path.strip }
       if @paths.empty?
-        raise ConfigError, "tail: 'path' parameter is required on tail input"
+        raise Fluent::ConfigError, "tail: 'path' parameter is required on tail input"
       end
 
       unless @pos_file
@@ -93,21 +101,17 @@ module Fluent
         $log.warn "this parameter is highly recommended to save the position to resume tailing."
       end
 
-      configure_parser(conf)
       configure_tag
+      configure_encoding
 
-      @multiline_mode = conf['format'] =~ /multiline/
+      @multiline_mode = parser_config["@type"] =~ /multiline/
       @receive_handler = if @multiline_mode
                            method(:parse_multilines)
                          else
                            method(:parse_singleline)
                          end
       @file_perm = system_config.file_permission || FILE_PERMISSION
-    end
-
-    def configure_parser(conf)
-      @parser = Plugin.new_parser(conf['format'])
-      @parser.configure(conf)
+      @parser = parser_create(conf: parser_config)
     end
 
     def configure_tag
@@ -120,6 +124,25 @@ module Fluent
       end
     end
 
+    def configure_encoding
+      unless @encoding
+        if @from_encoding
+          raise Fluent::ConfigError, "tail: 'from_encoding' parameter must be specified with 'encoding' parameter."
+        end
+      end
+
+      @encoding = parse_encoding_param(@encoding) if @encoding
+      @from_encoding = parse_encoding_param(@from_encoding) if @from_encoding
+    end
+
+    def parse_encoding_param(encoding_name)
+      begin
+        Encoding.find(encoding_name) if encoding_name
+      rescue ArgumentError => e
+        raise Fluent::ConfigError, e.message
+      end
+    end
+
     def start
       super
 
@@ -129,20 +152,12 @@ module Fluent
         @pf = PositionFile.parse(@pf_file)
       end
 
-      @loop = Coolio::Loop.new
       refresh_watchers
-
-      @refresh_trigger = TailWatcher::TimerWatcher.new(@refresh_interval, true, log, &method(:refresh_watchers))
-      @refresh_trigger.attach(@loop)
-      @thread = Thread.new(&method(:run))
+      timer_execute(:in_tail_refresh_watchers, @refresh_interval, &method(:refresh_watchers))
     end
 
     def shutdown
-      @refresh_trigger.detach if @refresh_trigger && @refresh_trigger.attached?
-
       stop_watchers(@tails.keys, true)
-      @loop.stop rescue nil # when all watchers are detached, `stop` raises RuntimeError. We can ignore this exception.
-      @thread.join
       @pf_file.close if @pf_file
 
       super
@@ -190,8 +205,11 @@ module Fluent
 
     def setup_watcher(path, pe)
       line_buffer_timer_flusher = (@multiline_mode && @multiline_flush_interval) ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
-      tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @enable_watch_timer, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher,  &method(:receive_lines))
-      tw.attach(@loop)
+      tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @enable_watch_timer, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher, &method(:receive_lines))
+      tw.attach do |watcher|
+        timer_execute(:in_tail_timer_trigger, 1, &watcher.method(:on_notify)) if watcher.enable_watch_timer
+        event_loop_attach(watcher.stat_trigger)
+      end
       tw
     end
 
@@ -202,7 +220,7 @@ module Fluent
           pe = @pf[path]
           if @read_from_head && pe.read_inode.zero?
             begin
-              pe.update(FileWrapper.stat(path).ino, 0)
+              pe.update(Fluent::FileWrapper.stat(path).ino, 0)
             rescue Errno::ENOENT
               $log.warn "#{path} not found. Continuing without tailing it."
             end
@@ -247,14 +265,21 @@ module Fluent
     end
 
     def close_watcher_after_rotate_wait(tw)
-      closer = TailWatcher::Closer.new(@rotate_wait, tw, log, &method(:close_watcher))
-      closer.attach(@loop)
+      timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
+        close_watcher(tw)
+      end
     end
 
     def flush_buffer(tw)
       if lb = tw.line_buffer
         lb.chomp!
-        lb.force_encoding(@encoding) if @encoding
+        if @encoding
+          if @from_encoding
+            lb.encode!(@encoding, @from_encoding)
+          else
+            lb.force_encoding(@encoding)
+          end
+        end
         @parser.parse(lb) { |time, record|
           if time && record
             tag = if @tag_prefix || @tag_suffix
@@ -271,13 +296,6 @@ module Fluent
       end
     end
 
-    def run
-      @loop.run
-    rescue
-      log.error "unexpected error", error: $!.to_s
-      log.error_backtrace
-    end
-
     # @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
     def receive_lines(lines, tail_watcher)
       es = @receive_handler.call(lines, tail_watcher)
@@ -303,7 +321,13 @@ module Fluent
     def convert_line_to_event(line, es, tail_watcher)
       begin
         line.chomp!  # remove \n
-        line.force_encoding(@encoding) if @encoding
+        if @encoding
+          if @from_encoding
+            line.encode!(@encoding, @from_encoding)
+          else
+            line.force_encoding(@encoding)
+          end
+        end
         @parser.parse(line) { |time, record|
           if time && record
             record[@path_key] ||= tail_watcher.path unless @path_key.nil?
@@ -319,7 +343,7 @@ module Fluent
     end
 
     def parse_singleline(lines, tail_watcher)
-      es = MultiEventStream.new
+      es = Fluent::MultiEventStream.new
       lines.each { |line|
         convert_line_to_event(line, es, tail_watcher)
       }
@@ -328,7 +352,7 @@ module Fluent
 
     def parse_multilines(lines, tail_watcher)
       lb = tail_watcher.line_buffer
-      es = MultiEventStream.new
+      es = Fluent::MultiEventStream.new
       if @parser.has_firstline?
         tail_watcher.line_buffer_timer_flusher.reset_timer if tail_watcher.line_buffer_timer_flusher
         lines.each { |line|
@@ -372,8 +396,6 @@ module Fluent
         @receive_lines = receive_lines
         @update_watcher = update_watcher
 
-        @timer_trigger = TimerWatcher.new(1, true, log, &method(:on_notify)) if @enable_watch_timer
-
         @stat_trigger = StatWatcher.new(path, log, &method(:on_notify))
 
         @rotate_handler = RotateHandler.new(path, log, &method(:on_rotate))
@@ -384,6 +406,8 @@ module Fluent
       end
 
       attr_reader :path
+      attr_reader :stat_trigger, :enable_watch_timer
+      attr_accessor :timer_trigger
       attr_accessor :line_buffer, :line_buffer_timer_flusher
       attr_accessor :unwatched  # This is used for removing position entry from PositionFile
 
@@ -395,14 +419,12 @@ module Fluent
         @receive_lines.call(lines, self)
       end
 
-      def attach(loop)
-        @timer_trigger.attach(loop) if @enable_watch_timer
-        @stat_trigger.attach(loop)
+      def attach
+        yield self
         on_notify
       end
 
       def detach
-        @timer_trigger.detach if @enable_watch_timer && @timer_trigger.attached?
         @stat_trigger.detach if @stat_trigger.attached?
       end
 
@@ -495,22 +517,6 @@ module Fluent
         pe # This pe will be updated in on_rotate after TailWatcher is initialized
       end
 
-      class TimerWatcher < Coolio::TimerWatcher
-        def initialize(interval, repeat, log, &callback)
-          @callback = callback
-          @log = log
-          super(interval, repeat)
-        end
-
-        def on_timer
-          @callback.call
-        rescue
-          # TODO log?
-          @log.error $!.to_s
-          @log.error_backtrace
-        end
-      end
-
       class StatWatcher < Coolio::StatWatcher
         def initialize(path, log, &callback)
           @callback = callback
@@ -527,24 +533,6 @@ module Fluent
         end
       end
 
-      class Closer < Coolio::TimerWatcher
-        def initialize(interval, tw, log, &callback)
-          @callback = callback
-          @tw = tw
-          @log = log
-          super(interval, false)
-        end
-
-        def on_timer
-          @callback.call(@tw)
-        rescue => e
-          @log.error e.to_s
-          @log.error_backtrace(e.backtrace)
-        ensure
-          detach
-        end
-      end
-
       class IOHandler
         def initialize(io, pe, log, read_lines_limit, first = true, &receive_lines)
           @log = log
@@ -632,7 +620,7 @@ module Fluent
 
         def on_notify
           begin
-            stat = FileWrapper.stat(@path)
+            stat = Fluent::FileWrapper.stat(@path)
             inode = stat.ino
             fsize = stat.size
           rescue Errno::ENOENT
@@ -645,7 +633,7 @@ module Fluent
             if @inode != inode || fsize < @fsize
               # rotated or truncated
               begin
-                io = FileWrapper.open(@path)
+                io = Fluent::FileWrapper.open(@path)
               rescue Errno::ENOENT
               end
               @on_rotate.call(io)
@@ -660,7 +648,6 @@ module Fluent
         end
       end
 
-
       class LineBufferTimerFlusher
         def initialize(log, flush_interval, &flush_method)
           @log = log
@@ -685,7 +672,6 @@ module Fluent
       end
     end
 
-
     class PositionFile
       UNWATCHED_POSITION = 0xffffffffffffffff
 
@@ -805,6 +791,4 @@ module Fluent
       end
     end
   end
-
-  NewTailInput = TailInput # for backward compatibility
 end

data/lib/fluent/plugin/multi_output.rb

@@ -53,6 +53,10 @@ module Fluent
         # @rollback_count = 0
       end
 
+      def multi_output?
+        true
+      end
+
       def configure(conf)
         super