fluentd 0.12.40 → 1.6.2

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -18,344 +18,255 @@ require 'json'
 require 'webrick'
 require 'cgi'
 
-require 'cool.io'
+require 'fluent/config/types'
+require 'fluent/plugin/input'
+require 'fluent/plugin/output'
+require 'fluent/plugin/multi_output'
+require 'fluent/plugin/filter'
 
-require 'fluent/input'
-require 'fluent/output'
-require 'fluent/filter'
-
-module Fluent
+module Fluent::Plugin
   class MonitorAgentInput < Input
-    Plugin.register_input('monitor_agent', self)
+    Fluent::Plugin.register_input('monitor_agent', self)
+
+    helpers :timer, :thread, :http_server
 
+    desc 'The address to bind to.'
     config_param :bind, :string, default: '0.0.0.0'
+    desc 'The port to listen to.'
     config_param :port, :integer, default: 24220
+    desc 'The tag with which internal metrics are emitted.'
     config_param :tag, :string, default: nil
+    desc 'Determine the rate to emit internal metrics as events.'
     config_param :emit_interval, :time, default: 60
-    config_param :emit_config, :bool, default: false
+    desc 'Determine whether to include the config information.'
     config_param :include_config, :bool, default: true
+    desc 'Determine whether to include the retry information.'
     config_param :include_retry, :bool, default: true
 
-    class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet
-      def initialize(server, agent)
+    class APIHandler
+      def initialize(agent)
         @agent = agent
       end
 
-      def do_GET(req, res)
-        begin
-          code, header, body = process(req, res)
-        rescue
-          code, header, body = render_json_error(500, {
-              'message '=> 'Internal Server Error',
-              'error' => "#{$!}",
-              'backgrace'=> $!.backtrace,
-            })
-        end
+      def plugins_ltsv(req)
+        list = build_object(build_option(req))
 
-        # set response code, header and body
-        res.status = code
-        header.each_pair {|k,v|
-          res[k] = v
-        }
-        res.body = body
+        render_ltsv(list)
       end
 
-      def build_object(req, res)
-        unless req.path_info == ""
-          return render_json_error(404, "Not found")
-        end
+      def plugins_json(req)
+        opts = build_option(req)
+        obj = build_object(opts)
 
-        # parse ?=query string
-        if req.query_string
-          begin
-            qs = CGI.parse(req.query_string)
-          rescue
-            return render_json_error(400, "Invalid query string")
+        render_json({ 'plugins' => obj }, pretty_json: opts[:pretty_json])
+      end
+
+      def config_ltsv(_req)
+        obj = {
+          'pid' => Process.pid,
+          'ppid' => Process.ppid
+        }.merge(@agent.fluentd_opts)
+
+        render_ltsv([obj])
+      end
+
+      def config_json(req)
+        obj = {
+          'pid' => Process.pid,
+          'ppid' => Process.ppid
+        }.merge(@agent.fluentd_opts)
+        opts = build_option(req)
+
+        render_json(obj, pretty_json: opts[:pretty_json])
+      end
+
+      private
+
+      def render_error_json(code:, msg:, pretty_json: nil, **additional_params)
+        resp = additional_params.merge('message' => msg)
+        render_json(resp, code: code, pretty_json: pretty_json)
+      end
+
+      def render_json(obj, code: 200, pretty_json: nil)
+        body =
+          if pretty_json
+            JSON.pretty_generate(obj)
+          else
+            obj.to_json
           end
-        else
-          qs = Hash.new {|h,k| [] }
-        end
 
-        # if ?debug=1 is set, set :with_debug_info for get_monitor_info
-        # and :pretty_json for render_json_error
-        opts = {with_config: @agent.include_config, with_retry: @agent.include_retry}
-        if s = qs['debug'] and s[0]
-          opts[:with_debug_info] = true
-          opts[:pretty_json] = true
-        end
+        [code, { 'Content-Type' => 'application/json' }, body]
+      end
 
-        if ivars = (qs['with_ivars'] || []).first
-          opts[:ivars] = ivars.split(',')
-        end
+      def render_ltsv(obj, code: 200)
+        normalized = JSON.parse(obj.to_json)
+        text = ''
+        normalized.each do |hash|
+          row = []
+          hash.each do |k, v|
+            if v.is_a?(Array)
+              row << "#{k}:#{v.join(',')}"
+            elsif v.is_a?(Hash)
+              next
+            else
+              row << "#{k}:#{v}"
+            end
+          end
 
-        if with_config = get_search_parameter(qs, 'with_config'.freeze)
-          opts[:with_config] = Fluent::Config.bool_value(with_config)
+          text << row.join("\t") << "\n"
         end
 
-        if with_retry = get_search_parameter(qs, 'with_retry'.freeze)
-          opts[:with_retry] = Fluent::Config.bool_value(with_retry)
-        end
+        [code, { 'Content-Type' => 'text/plain' }, text]
+      end
 
-        if tag = get_search_parameter(qs, 'tag'.freeze)
+      def build_object(opts)
+        qs = opts[:query]
+        if tag = qs['tag'.freeze].first
           # ?tag= to search an output plugin by match pattern
           if obj = @agent.plugin_info_by_tag(tag, opts)
             list = [obj]
           else
             list = []
           end
-
-        elsif plugin_id = get_search_parameter(qs, '@id'.freeze)
+        elsif plugin_id = (qs['@id'.freeze].first || qs['id'.freeze].first)
           # ?@id= to search a plugin by 'id <plugin_id>' config param
           if obj = @agent.plugin_info_by_id(plugin_id, opts)
             list = [obj]
           else
             list = []
           end
-
-        elsif plugin_id = get_search_parameter(qs, 'id'.freeze)
-          # Without @ version of ?@id= for backward compatibility
-          if obj = @agent.plugin_info_by_id(plugin_id, opts)
-            list = [obj]
-          else
-            list = []
-          end
-
-        elsif plugin_type = get_search_parameter(qs, '@type'.freeze)
+        elsif plugin_type = (qs['@type'.freeze].first || qs['type'.freeze].first)
           # ?@type= to search plugins by 'type <type>' config param
           list = @agent.plugins_info_by_type(plugin_type, opts)
-
-        elsif plugin_type = get_search_parameter(qs, 'type'.freeze)
-          # Without @ version of ?@type= for backward compatibility
-          list = @agent.plugins_info_by_type(plugin_type, opts)
-
         else
           # otherwise show all plugins
           list = @agent.plugins_info_all(opts)
         end
 
-        return list, opts
-      end
-
-      def get_search_parameter(qs, param_name)
-        return nil unless qs.has_key?(param_name)
-        qs[param_name].first
+        list
       end
 
-      def render_json(obj, opts={})
-        render_json_error(200, obj, opts)
-      end
-
-      def render_json_error(code, obj, opts={})
-        if opts[:pretty_json]
-          js = JSON.pretty_generate(obj)
-        else
-          js = obj.to_json
+      def build_option(req)
+        qs = Hash.new { |_, _| [] }
+        # parse ?=query string
+        if req.query_string
+          qs.merge!(CGI.parse(req.query_string))
         end
-        [code, {'Content-Type'=>'application/json'}, js]
-      end
-    end
 
-    class LTSVMonitorServlet < MonitorServlet
-      def process(req, res)
-        list, opts = build_object(req, res)
-        return unless list
+        # if ?debug=1 is set, set :with_debug_info for get_monitor_info
+        # and :pretty_json for render_json_error
+        opts = { query: qs }
+        if qs['debug'.freeze].first
+          opts[:with_debug_info] = true
+          opts[:pretty_json] = true
+        end
 
-        normalized = JSON.parse(list.to_json)
+        if ivars = qs['with_ivars'.freeze].first
+          opts[:ivars] = ivars.split(',')
+        end
 
-        text = ''
+        if with_config = qs['with_config'.freeze].first
+          opts[:with_config] = Fluent::Config.bool_value(with_config)
+        else
+          opts[:with_config] = @agent.include_config
+        end
 
-        normalized.map {|hash|
-          row = []
-          hash.each_pair {|k,v|
-            unless v.is_a?(Hash) || v.is_a?(Array)
-              row << "#{k}:#{v}"
-            end
-          }
-          text << row.join("\t") << "\n"
-        }
+        if with_retry = qs['with_retry'.freeze].first
+          opts[:with_retry] = Fluent::Config.bool_value(with_retry)
+        else
+          opts[:with_retry] = @agent.include_retry
+        end
 
-        [200, {'Content-Type'=>'text/plain'}, text]
+        opts
       end
     end
 
-    class JSONMonitorServlet < MonitorServlet
-      def process(req, res)
-        list, opts = build_object(req, res)
-        return unless list
+    def initialize
+      super
 
-        render_json({
-            'plugins' => list
-          }, opts)
-      end
+      @first_warn = false
     end
 
-    class ConfigMonitorServlet < MonitorServlet
-      def build_object(req, res)
-        {
-          'pid' => Process.pid,
-          'ppid' => Process.ppid
-        }.merge(@agent.fluentd_opts)
-      end
+    def configure(conf)
+      super
+      @port += fluentd_worker_id
     end
 
-    class LTSVConfigMonitorServlet < ConfigMonitorServlet
-      def process(req, res)
-        result = build_object(req, res)
-
-        row = []
-        JSON.parse(result.to_json).each_pair { |k, v|
-          row << "#{k}:#{v}"
-        }
-        text = row.join("\t")
-
-        [200, {'Content-Type'=>'text/plain'}, text]
-      end
+    def multi_workers_ready?
+      true
     end
 
-    class JSONConfigMonitorServlet < ConfigMonitorServlet
-      def process(req, res)
-        result = build_object(req, res)
-        render_json(result)
+    class NotFoundJson
+      BODY = { 'message' => 'Not found' }.to_json
+      def self.call(_req)
+        [404, { 'Content-Type' => 'application/json' }, BODY]
       end
     end
 
-    class TimerWatcher < Coolio::TimerWatcher
-      def initialize(interval, log, &callback)
-        @callback = callback
-        @log = log
-
-        # Avoid long shutdown time
-        @num_call = 0
-        if interval >= 10
-          min_interval = 10
-          @call_interval = interval / 10
-        else
-          min_interval = interval
-          @call_interval = 0
-        end
-
-        super(min_interval, true)
-      end
-
-      def on_timer
-        @num_call += 1
-        if @num_call >= @call_interval
-          @num_call = 0
-          @callback.call
-        end
-      rescue => e
-        @log.error e.to_s
-        @log.error_backtrace
+    def start
+      super
+
+      log.debug "listening monitoring http server on http://#{@bind}:#{@port}/api/plugins for worker#{fluentd_worker_id}"
+      api_handler = APIHandler.new(self)
+      create_http_server(:in_monitor_http_server_helper, addr: @bind, port: @port, logger: log, default_app: NotFoundJson) do |serv|
+        serv.get('/api/plugins') { |req| api_handler.plugins_ltsv(req) }
+        serv.get('/api/plugins.json') { |req| api_handler.plugins_json(req) }
+        serv.get('/api/config') { |req| api_handler.config_ltsv(req) }
+        serv.get('/api/config.json') { |req| api_handler.config_json(req) }
       end
-    end
 
-    def start
-      log.debug "listening monitoring http server on http://#{@bind}:#{@port}/api/plugins"
-      @srv = WEBrick::HTTPServer.new({
-          BindAddress: @bind,
-          Port: @port,
-          Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
-          AccessLog: [],
-        })
-      @srv.mount('/api/plugins', LTSVMonitorServlet, self)
-      @srv.mount('/api/plugins.json', JSONMonitorServlet, self)
-      @srv.mount('/api/config', LTSVConfigMonitorServlet, self)
-      @srv.mount('/api/config.json', JSONConfigMonitorServlet, self)
-      @thread = Thread.new {
-        @srv.start
-      }
       if @tag
         log.debug "tag parameter is specified. Emit plugins info to '#{@tag}'"
 
-        @loop = Coolio::Loop.new
-        opts = {with_config: @emit_config, with_retry: false}
-        timer = TimerWatcher.new(@emit_interval, log) {
-          es = MultiEventStream.new
-          now = Engine.now
+        opts = {with_config: false, with_retry: false}
+        timer_execute(:in_monitor_agent_emit, @emit_interval, repeat: true) {
+          es = Fluent::MultiEventStream.new
+          now = Fluent::Engine.now
           plugins_info_all(opts).each { |record|
             es.add(now, record)
           }
           router.emit_stream(@tag, es)
         }
-        @loop.attach(timer)
-        @thread_for_emit = Thread.new(&method(:run))
-      end
-    end
-
-    def run
-      @loop.run
-    rescue => e
-      log.error "unexpected error", error: e.to_s
-      log.error_backtrace
-    end
-
-    def shutdown
-      if @srv
-        @srv.shutdown
-        @srv = nil
-      end
-      if @thread
-        @thread.join
-        @thread = nil
-      end
-      if @tag
-        @loop.watchers.each { |w| w.detach }
-        @loop.stop
-        @loop = nil
-        @thread_for_emit.join
-        @thread_for_emit = nil
       end
     end
 
+    # They are deprecated but remain for compatibility
     MONITOR_INFO = {
-      'output_plugin' => 'is_a?(::Fluent::Output)', # deprecated. Use plugin_category instead
-      'buffer_queue_length' => '@buffer.queue_size',
-      'buffer_total_queued_size' => '@buffer.total_queued_chunk_size',
-      'retry_count' => '@num_errors',
+      'output_plugin' => ->(){ is_a?(::Fluent::Plugin::Output) },
+      'buffer_queue_length' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.queue.size },
+      'buffer_timekeys' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.timekeys },
+      'buffer_total_queued_size' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.stage_size + @buffer.queue_size },
+      'retry_count' => ->(){ instance_variable_defined?(:@num_errors) ? @num_errors : nil },
     }
 
     def all_plugins
       array = []
 
       # get all input plugins
-      array.concat Engine.root_agent.inputs
+      array.concat Fluent::Engine.root_agent.inputs
 
       # get all output plugins
-      Engine.root_agent.outputs.each { |o|
-        MonitorAgentInput.collect_children(o, array)
-      }
+      array.concat Fluent::Engine.root_agent.outputs
+
       # get all filter plugins
-      Engine.root_agent.filters.each { |f|
-        MonitorAgentInput.collect_children(f, array)
-      }
-      Engine.root_agent.labels.each { |name, l|
+      array.concat Fluent::Engine.root_agent.filters
+
+      Fluent::Engine.root_agent.labels.each { |name, l|
         # TODO: Add label name to outputs / filters for identifing plugins
-        l.outputs.each { |o| MonitorAgentInput.collect_children(o, array) }
-        l.filters.each { |f| MonitorAgentInput.collect_children(f, array) }
+        array.concat l.outputs
+        array.concat l.filters
       }
 
       array
     end
 
-    # get nexted plugins (such as <store> of the copy plugin)
-    # from the plugin `pe` recursively
-    def self.collect_children(pe, array=[])
-      array << pe
-      if pe.is_a?(MultiOutput) && pe.respond_to?(:outputs)
-        pe.outputs.each {|nop|
-          collect_children(nop, array)
-        }
-      end
-      array
-    end
-
     # try to match the tag and get the info from the matched output plugin
     # TODO: Support output in label
     def plugin_info_by_tag(tag, opts={})
-      matches = Engine.root_agent.event_router.instance_variable_get(:@match_rules)
+      matches = Fluent::Engine.root_agent.event_router.instance_variable_get(:@match_rules)
       matches.each { |rule|
         if rule.match?(tag)
-          if rule.collector.is_a?(Output)
+          if rule.collector.is_a?(Fluent::Plugin::Output) || rule.collector.is_a?(Fluent::Output)
             return get_monitor_info(rule.collector, opts)
           end
         end
@@ -379,7 +290,7 @@ module Fluent
     # multiple plugins could have the same type
     def plugins_info_by_type(type, opts={})
       array = all_plugins.select {|pe|
-        (pe.config['@type'] == type || pe.config['type'] == type || Fluent::Plugin.lookup_name_from_class(pe.class)) rescue nil
+        (pe.config['@type'] == type) rescue nil
       }
       array.map {|pe|
         get_monitor_info(pe, opts)
@@ -392,9 +303,7 @@ module Fluent
       }
     end
 
-    # TODO: use %i() after drop ruby v1.9.3 support.
-    IGNORE_ATTRIBUTES = %W(@config_root_section @config @masked_config).map(&:to_sym)
-    EMPTY_RESULT = {}
+    IGNORE_ATTRIBUTES = %i(@config_root_section @config @masked_config)
 
     # get monitor info from the plugin `pe` and return a hash object
     def get_monitor_info(pe, opts={})
@@ -403,24 +312,32 @@ module Fluent
       # Common plugin information
       obj['plugin_id'] = pe.plugin_id
       obj['plugin_category'] = plugin_category(pe)
-      obj['type'] = pe.config['@type'] || pe.config['type'] || Fluent::Plugin.lookup_name_from_class(pe.class)
+      obj['type'] = pe.config['@type']
       obj['config'] = pe.config if opts[:with_config]
 
       # run MONITOR_INFO in plugins' instance context and store the info to obj
       MONITOR_INFO.each_pair {|key,code|
         begin
-          obj[key] = pe.instance_eval(code)
-        rescue
+          catch(:skip) do
+            obj[key] = pe.instance_exec(&code)
+          end
+        rescue NoMethodError => e
+          unless @first_warn
+            log.error "NoMethodError in monitoring plugins", key: key, plugin: pe.class, error: e
+            log.error_backtrace
+            @first_warn = true
+          end
+        rescue => e
+          log.warn "unexpected error in monitoring plugins", key: key, plugin: pe.class, error: e
         end
       }
 
-      if opts[:with_retry]
-        num_errors = pe.instance_variable_get(:@num_errors)
-        if num_errors
-          obj['retry'] = num_errors.zero? ? EMPTY_RESULT : get_retry_info(pe, num_errors)
-        end
+      if pe.respond_to?(:statistics)
+        obj.merge!(pe.statistics['output'] || {})
       end
 
+      obj['retry'] = get_retry_info(pe.retry) if opts[:with_retry] and pe.instance_variable_defined?(:@retry)
+
       # include all instance variables if :with_debug_info is set
       if opts[:with_debug_info]
         iv = {}
@@ -444,20 +361,31 @@ module Fluent
       obj
     end
 
-    def get_retry_info(pe, num_errors)
+    RETRY_INFO = {
+        'start' => '@start',
+        'steps' => '@steps',
+        'next_time' => '@next_time',
+    }
+
+    def get_retry_info(pe_retry)
       retry_variables = {}
-      retry_variables['steps'] = num_errors
-      retry_variables['next_time'] = Time.at(pe.instance_variable_get('@next_retry_time'.freeze))
+
+      if pe_retry
+        RETRY_INFO.each_pair { |key, param|
+          retry_variables[key] = pe_retry.instance_variable_get(param)
+        }
+      end
+
       retry_variables
     end
 
     def plugin_category(pe)
       case pe
-      when Fluent::Input
+      when Fluent::Plugin::Input
         'input'.freeze
-      when Fluent::Output
+      when Fluent::Plugin::Output, Fluent::Plugin::MultiOutput, Fluent::Plugin::BareOutput
         'output'.freeze
-      when Fluent::Filter
+      when Fluent::Plugin::Filter
         'filter'.freeze
       else
         'unknown'.freeze

data/lib/fluent/plugin/in_object_space.rb

@@ -14,14 +14,15 @@
 #    limitations under the License.
 #
 
-require 'cool.io'
 require 'yajl'
 
-require 'fluent/input'
+require 'fluent/plugin/input'
 
-module Fluent
-  class ObjectSpaceInput < Input
-    Plugin.register_input('object_space', self)
+module Fluent::Plugin
+  class ObjectSpaceInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('object_space', self)
+
+    helpers :timer
 
     def initialize
       super
@@ -31,44 +32,14 @@ module Fluent
     config_param :tag, :string
     config_param :top, :integer, default: 15
 
-    class TimerWatcher < Coolio::TimerWatcher
-      def initialize(interval, repeat, log, &callback)
-        @callback = callback
-        @log = log
-        super(interval, repeat)
-      end
-
-      def on_timer
-        @callback.call
-      rescue
-        # TODO log?
-        @log.error $!.to_s
-        @log.error_backtrace
-      end
-    end
-
-    def configure(conf)
-      super
+    def multi_workers_ready?
+      true
     end
 
     def start
-      @loop = Coolio::Loop.new
-      @timer = TimerWatcher.new(@emit_interval, true, log, &method(:on_timer))
-      @loop.attach(@timer)
-      @thread = Thread.new(&method(:run))
-    end
-
-    def shutdown
-      @loop.watchers.each {|w| w.detach }
-      @loop.stop
-      @thread.join
-    end
+      super
 
-    def run
-      @loop.run
-    rescue
-      log.error "unexpected error", error: $!.to_s
-      log.error_backtrace
+      timer_execute(:object_space_input, @emit_interval, &method(:on_timer))
     end
 
     class Counter
@@ -89,7 +60,7 @@ module Fluent
     end
 
     def on_timer
-      now = Engine.now
+      now = Fluent::EventTime.now
 
       array = []
       map = {}
@@ -115,7 +86,8 @@ module Fluent
 
       router.emit(@tag, now, record)
     rescue => e
-      log.error "object space failed to emit", error: e.to_s, error_class: e.class.to_s, tag: @tag, record: Yajl.dump(record)
+      log.error "object space failed to emit", error: e, tag: @tag, record: Yajl.dump(record)
+      log.error_backtrace
     end
   end
 end