fluentd 0.14.3 → 0.14.4

data/lib/fluent/plugin/filter.rb

@@ -30,22 +30,64 @@ module Fluent
 
       helpers :event_emitter
 
+      attr_reader :has_filter_with_time
+
+      def initialize
+        super
+        @has_filter_with_time = has_filter_with_time?
+      end
+
       def filter(tag, time, record)
         raise NotImplementedError, "BUG: filter plugins MUST implement this method"
       end
 
+      def filter_with_time(tag, time, record)
+        raise NotImplementedError, "BUG: filter plugins MUST implement this method"
+      end
+
       def filter_stream(tag, es)
         new_es = MultiEventStream.new
-        es.each do |time, record|
-          begin
-            filtered_record = filter(tag, time, record)
-            new_es.add(time, filtered_record) if filtered_record
-          rescue => e
-            router.emit_error_event(tag, time, record, e)
+        if @has_filter_with_time
+          es.each do |time, record|
+            begin
+              filtered_time, filtered_record = filter_with_time(tag, time, record)
+              new_es.add(filtered_time, filtered_record) if filtered_time && filtered_record
+            rescue => e
+              router.emit_error_event(tag, time, record, e)
+            end
+          end
+        else
+          es.each do |time, record|
+            begin
+              filtered_record = filter(tag, time, record)
+              new_es.add(time, filtered_record) if filtered_record
+            rescue => e
+              router.emit_error_event(tag, time, record, e)
+            end
           end
         end
         new_es
       end
+
+      private
+
+      def has_filter_with_time?
+        implmented_methods = self.class.instance_methods(false)
+        # Plugins that override `filter_stream` don't need check,
+        # because they may not call `filter` or `filter_with_time`
+        # for example fluentd/lib/fluent/plugin/filter_record_transformer.rb
+        return nil if implmented_methods.include?(:filter_stream)
+        case
+        when [:filter, :filter_with_time].all? { |e| implmented_methods.include?(e) }
+          raise "BUG: Filter plugins MUST be implemented either `filter` or `filter_with_time`"
+        when implmented_methods.include?(:filter)
+          false
+        when implmented_methods.include?(:filter_with_time)
+          true
+        else
+          raise NotImplementedError, "BUG: Filter plugins MUST be implmented either `filter` or `filter_with_time`"
+        end
+      end
     end
   end
 end

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -204,7 +204,9 @@ module Fluent::Plugin
             end
           elsif value.kind_of?(Hash) # record, etc
             value.each do |k, v|
-              placeholders.store("${#{k}}", v) # foo
+              unless placeholder_values.has_key?(k) # prevent overwriting reserved keys such as tag
+                placeholders.store("${#{k}}", v) # foo
+              end
               placeholders.store(%Q[${#{key}["#{k}"]}], v) # record["foo"]
             end
           else # string, interger, float, and others?

data/lib/fluent/plugin/filter_stdout.rb

@@ -34,11 +34,7 @@ module Fluent::Plugin
     def configure(conf)
       compat_parameters_convert(conf, :inject, :formatter)
       super
-    end
-
-    def start
       @formatter = formatter_create(conf: @config.elements('format').first, default_type: DEFAULT_FORMAT_TYPE)
-      super
     end
 
     def filter_stream(tag, es)

data/lib/fluent/plugin/in_debug_agent.rb

@@ -14,11 +14,11 @@
 #    limitations under the License.
 #
 
-require 'fluent/input'
+require 'fluent/plugin/input'
 
-module Fluent
+module Fluent::Plugin
   class DebugAgentInput < Input
-    Plugin.register_input('debug_agent', self)
+    Fluent::Plugin.register_input('debug_agent', self)
 
     def initialize
       require 'drb/drb'
@@ -30,13 +30,13 @@ module Fluent
     config_param :port, :integer, default: 24230
     config_param :unix_path, :string, default: nil
     #config_param :unix_mode  # TODO
-    config_param :object, :string, default: 'Engine'
+    config_param :object, :string, default: 'Fluent::Engine'
 
     def configure(conf)
       super
       if @unix_path
         unless ::Fluent::FileUtil.writable?(@unix_path)
-          raise ConfigError, "in_debug_agent: `#{@unix_path}` is not writable"
+          raise Fluent::ConfigError, "in_debug_agent: `#{@unix_path}` is not writable"
         end
       end
     end

data/lib/fluent/plugin/in_dummy.rb

@@ -30,6 +30,8 @@ module Fluent::Plugin
 
     desc "The value is the tag assigned to the generated events."
     config_param :tag, :string
+    desc "The number of events in event stream of each emits."
+    config_param :size, :integer, default: 1
     desc "It configures how many events to generate per second."
     config_param :rate, :integer, default: 1
     desc "If specified, each generated event has an auto-incremented key field."
@@ -97,7 +99,13 @@ module Fluent::Plugin
 
     def emit(num)
       begin
-        num.times { router.emit(@tag, Fluent::Engine.now, generate()) }
+        if @size > 1
+          num.times do
+            router.emit_array(@tag, Array.new(@size) { [Fluent::Engine.now, generate] })
+          end
+        else
+          num.times { router.emit(@tag, Fluent::Engine.now, generate) }
+        end
       rescue => _
         # ignore all errors not to stop emits by emit errors
       end

data/lib/fluent/plugin/in_forward.rb

@@ -49,6 +49,8 @@ module Fluent
     config_param :chunk_size_limit, :size, default: nil
     desc 'Skip an event if incoming event is invalid.'
     config_param :skip_invalid_event, :bool, default: false
+    desc "The field name of the client's hostname."
+    config_param :source_hostname_key, :string, default: nil
 
     def configure(conf)
       super
@@ -145,7 +147,7 @@ module Fluent
     #   3: object record
     #   4: object option (optional)
     # }
-    def on_message(msg, chunk_size, source)
+    def on_message(msg, chunk_size, peeraddr)
       if msg.nil?
         # for future TCP heartbeat_request
         return
@@ -153,7 +155,7 @@ module Fluent
 
       # TODO: raise an exception if broken chunk is generated by recoverable situation
       unless msg.is_a?(Array)
-        log.warn "incoming chunk is broken:", source: source, msg: msg
+        log.warn "incoming chunk is broken:", source: source_message(peeraddr), msg: msg
         return
       end
 
@@ -161,10 +163,10 @@ module Fluent
       entries = msg[1]
 
       if @chunk_size_limit && (chunk_size > @chunk_size_limit)
-        log.warn "Input chunk size is larger than 'chunk_size_limit', dropped:", tag: tag, source: source, limit: @chunk_size_limit, size: chunk_size
+        log.warn "Input chunk size is larger than 'chunk_size_limit', dropped:", tag: tag, source: source_message(peeraddr), limit: @chunk_size_limit, size: chunk_size
         return
       elsif @chunk_size_warn_limit && (chunk_size > @chunk_size_warn_limit)
-        log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag, source: source, limit: @chunk_size_warn_limit, size: chunk_size
+        log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag, source: source_message(peeraddr), limit: @chunk_size_warn_limit, size: chunk_size
       end
 
       if entries.class == String
@@ -172,13 +174,14 @@ module Fluent
         option = msg[2]
         size = (option && option['size']) || 0
         es = MessagePackEventStream.new(entries, nil, size.to_i)
-        es = check_and_skip_invalid_event(tag, es, source) if @skip_invalid_event
+        es = check_and_skip_invalid_event(tag, es, peeraddr) if @skip_invalid_event
+        es = add_source_host(es, peeraddr[2]) if @source_hostname_key
         router.emit_stream(tag, es)
 
       elsif entries.class == Array
         # Forward
         es = if @skip_invalid_event
-               check_and_skip_invalid_event(tag, entries, source)
+               check_and_skip_invalid_event(tag, entries, peeraddr)
              else
                es = MultiEventStream.new
                entries.each { |e|
@@ -190,6 +193,7 @@ module Fluent
                }
                es
              end
+        es = add_source_host(es, peeraddr[2]) if @source_hostname_key
         router.emit_stream(tag, es)
         option = msg[2]
 
@@ -198,11 +202,12 @@ module Fluent
         time = msg[1]
         record = msg[2]
         if @skip_invalid_event && invalid_event?(tag, time, record)
-          log.warn "got invalid event and drop it:", source: source, tag: tag, time: time, record: record
+          log.warn "got invalid event and drop it:", source: source_message(peeraddr), tag: tag, time: time, record: record
           return msg[3] # retry never succeeded so return ack and drop incoming event.
         end
         return if record.nil?
         time = Engine.now if time.to_i == 0
+        record[@source_hostname_key] = peeraddr[2] if @source_hostname_key
         router.emit(tag, time, record)
         option = msg[3]
       end
@@ -215,11 +220,11 @@ module Fluent
       !((time.is_a?(Integer) || time.is_a?(::Fluent::EventTime)) && record.is_a?(Hash) && tag.is_a?(String))
     end
 
-    def check_and_skip_invalid_event(tag, es, source)
+    def check_and_skip_invalid_event(tag, es, peeraddr)
       new_es = MultiEventStream.new
       es.each { |time, record|
         if invalid_event?(tag, time, record)
-          log.warn "skip invalid event:", source: source, tag: tag, time: time, record: record
+          log.warn "skip invalid event:", source: source_message(peeraddr), tag: tag, time: time, record: record
           next
         end
         new_es.add(time, record)
@@ -227,16 +232,29 @@ module Fluent
       new_es
     end
 
+    def add_source_host(es, host)
+      new_es = MultiEventStream.new
+      es.each { |time, record|
+        record[@source_hostname_key] = host
+        new_es.add(time, record)
+      }
+      new_es
+    end
+
+    def source_message(peeraddr)
+      _, port, host, addr = peeraddr
+      "host: #{host}, addr: #{addr}, port: #{port}"
+    end
+
     class Handler < Coolio::Socket
       PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
 
       def initialize(io, linger_timeout, log, on_message)
         super(io)
 
+        @peeraddr = nil
         if io.is_a?(TCPSocket) # for unix domain socket support in the future
-          _proto, port, host, addr = ( io.peeraddr rescue PEERADDR_FAILED )
-          @source = "host: #{host}, addr: #{addr}, port: #{port}"
-
+          @peeraddr = (io.peeraddr rescue PEERADDR_FAILED)
           opt = [1, linger_timeout].pack('I!I!')  # { int l_onoff; int l_linger; }
           io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
         end
@@ -265,7 +283,7 @@ module Fluent
           @serializer = :to_json.to_proc
           @y = Yajl::Parser.new
           @y.on_parse_complete = lambda { |obj|
-            option = @on_message.call(obj, @chunk_counter, @source)
+            option = @on_message.call(obj, @chunk_counter, @peeraddr)
             respond option
             @chunk_counter = 0
           }
@@ -293,7 +311,7 @@ module Fluent
       def on_read_msgpack(data)
         @chunk_counter += data.bytesize
         @u.feed_each(data) do |obj|
-          option = @on_message.call(obj, @chunk_counter, @source)
+          option = @on_message.call(obj, @chunk_counter, @peeraddr)
           respond option if option
           @chunk_counter = 0
         end

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -20,13 +20,16 @@ require 'cgi'
 
 require 'cool.io'
 
-require 'fluent/input'
-require 'fluent/output'
-require 'fluent/filter'
+require 'fluent/plugin/input'
+require 'fluent/plugin/output'
+require 'fluent/plugin/multi_output'
+require 'fluent/plugin/filter'
 
-module Fluent
+module Fluent::Plugin
   class MonitorAgentInput < Input
-    Plugin.register_input('monitor_agent', self)
+    Fluent::Plugin.register_input('monitor_agent', self)
+
+    helpers :timer, :thread
 
     config_param :bind, :string, default: '0.0.0.0'
     config_param :port, :integer, default: 24220
@@ -142,7 +145,7 @@ module Fluent
 
     class LTSVMonitorServlet < MonitorServlet
       def process(req, res)
-        list, opts = build_object(req, res)
+        list, _opts = build_object(req, res)
         return unless list
 
         normalized = JSON.parse(list.to_json)
@@ -204,36 +207,6 @@ module Fluent
       end
     end
 
-    class TimerWatcher < Coolio::TimerWatcher
-      def initialize(interval, log, &callback)
-        @callback = callback
-        @log = log
-
-        # Avoid long shutdown time
-        @num_call = 0
-        if interval >= 10
-          min_interval = 10
-          @call_interval = interval / 10
-        else
-          min_interval = interval
-          @call_interval = 0
-        end
-
-        super(min_interval, true)
-      end
-
-      def on_timer
-        @num_call += 1
-        if @num_call >= @call_interval
-          @num_call = 0
-          @callback.call
-        end
-      rescue => e
-        @log.error e.to_s
-        @log.error_backtrace
-      end
-    end
-
     def start
       super
 
@@ -248,76 +221,55 @@ module Fluent
       @srv.mount('/api/plugins.json', JSONMonitorServlet, self)
       @srv.mount('/api/config', LTSVConfigMonitorServlet, self)
       @srv.mount('/api/config.json', JSONConfigMonitorServlet, self)
-      @thread = Thread.new {
+      thread_create :in_monitor_agent_servlet do
         @srv.start
-      }
+      end
       if @tag
         log.debug "tag parameter is specified. Emit plugins info to '#{@tag}'"
 
-        @loop = Coolio::Loop.new
         opts = {with_config: false}
-        timer = TimerWatcher.new(@emit_interval, log) {
-          es = MultiEventStream.new
-          now = Engine.now
+        timer_execute(:in_monitor_agent_emit, @emit_interval, repeat: true) {
+          es = Fluent::MultiEventStream.new
+          now = Fluent::Engine.now
           plugins_info_all(opts).each { |record|
             es.add(now, record)
           }
           router.emit_stream(@tag, es)
         }
-        @loop.attach(timer)
-        @thread_for_emit = Thread.new(&method(:run))
       end
     end
 
-    def run
-      @loop.run
-    rescue => e
-      log.error "unexpected error", error: e.to_s
-      log.error_backtrace
-    end
-
     def shutdown
       if @srv
         @srv.shutdown
         @srv = nil
       end
-      if @thread
-        @thread.join
-        @thread = nil
-      end
-      if @tag
-        @loop.watchers.each { |w| w.detach }
-        @loop.stop
-        @loop = nil
-        @thread_for_emit.join
-        @thread_for_emit = nil
-      end
 
       super
     end
 
     MONITOR_INFO = {
-      'output_plugin' => 'is_a?(::Fluent::Plugin::Output)',
-      'buffer_queue_length' => '@buffer.queue.size',
-      'buffer_total_queued_size' => '@buffer.stage_size + @buffer.queue_size',
-      'retry_count' => '@num_errors',
+      'output_plugin' => ->(){ is_a?(::Fluent::Plugin::Output) },
+      'buffer_queue_length' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer); @buffer.queue.size },
+      'buffer_total_queued_size' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer); @buffer.stage_size },
+      'retry_count' => ->(){ instance_variable_defined?(:@num_errors) ? @num_errors : nil },
     }
 
     def all_plugins
       array = []
 
       # get all input plugins
-      array.concat Engine.root_agent.inputs
+      array.concat Fluent::Engine.root_agent.inputs
 
       # get all output plugins
-      Engine.root_agent.outputs.each { |o|
+      Fluent::Engine.root_agent.outputs.each { |o|
         MonitorAgentInput.collect_children(o, array)
       }
       # get all filter plugins
-      Engine.root_agent.filters.each { |f|
+      Fluent::Engine.root_agent.filters.each { |f|
         MonitorAgentInput.collect_children(f, array)
       }
-      Engine.root_agent.labels.each { |name, l|
+      Fluent::Engine.root_agent.labels.each { |name, l|
         # TODO: Add label name to outputs / filters for identifing plugins
         l.outputs.each { |o| MonitorAgentInput.collect_children(o, array) }
         l.filters.each { |f| MonitorAgentInput.collect_children(f, array) }
@@ -330,7 +282,7 @@ module Fluent
     # from the plugin `pe` recursively
     def self.collect_children(pe, array=[])
       array << pe
-      if pe.is_a?(MultiOutput) && pe.respond_to?(:outputs)
+      if pe.is_a?(Fluent::Plugin::MultiOutput) || pe.is_a?(Fluent::MultiOutput) && pe.respond_to?(:outputs)
         pe.outputs.each {|nop|
           collect_children(nop, array)
         }
@@ -341,10 +293,10 @@ module Fluent
     # try to match the tag and get the info from the matched output plugin
     # TODO: Support output in label
     def plugin_info_by_tag(tag, opts={})
-      matches = Engine.root_agent.event_router.instance_variable_get(:@match_rules)
+      matches = Fluent::Engine.root_agent.event_router.instance_variable_get(:@match_rules)
       matches.each { |rule|
         if rule.match?(tag)
-          if rule.collector.is_a?(Output)
+          if rule.collector.is_a?(Fluent::Plugin::Output) || rule.collector.is_a?(Fluent::Output)
             return get_monitor_info(rule.collector, opts)
           end
         end
@@ -381,8 +333,7 @@ module Fluent
       }
     end
 
-    # TODO: use %i() after drop ruby v1.9.3 support.
-    IGNORE_ATTRIBUTES = %W(@config_root_section @config @masked_config).map(&:to_sym)
+    IGNORE_ATTRIBUTES = %i(@config_root_section @config @masked_config)
 
     # get monitor info from the plugin `pe` and return a hash object
     def get_monitor_info(pe, opts={})
@@ -397,8 +348,11 @@ module Fluent
       # run MONITOR_INFO in plugins' instance context and store the info to obj
       MONITOR_INFO.each_pair {|key,code|
         begin
-          obj[key] = pe.instance_eval(code)
-        rescue
+          catch(:skip) do
+            obj[key] = pe.instance_exec(&code)
+          end
+        rescue => e
+          log.warn "unexpected error in monitoring plugins", key: key, plugin: pe.class, error: e
         end
       }