fluentd 0.14.1 → 0.14.2

data/lib/fluent/plugin/formatter_csv.rb

@@ -22,23 +22,22 @@ module Fluent
     class CsvFormatter < Formatter
       Plugin.register_formatter('csv', self)
 
-      include HandleTagAndTimeMixin
-
       config_param :delimiter, default: ',' do |val|
         ['\t', 'TAB'].include?(val) ? "\t" : val
       end
       config_param :force_quotes, :bool, default: true
       # "array" looks good for type of :fields, but this implementation removes tailing comma
       # TODO: Is it needed to support tailing comma?
-      config_param :fields, default: [] do |val|
-        val.split(',').map{|f| f.strip!; f.empty? ? nil : f }.compact
+      config_param :fields, :array, value_type: :string
+
+      def configure(conf)
+        super
+        @fields = fields.select{|f| !f.empty? }
       end
 
       def format(tag, time, record)
-        filter_record(tag, time, record)
-        row = @fields.inject([]) do |memo, key|
-            memo << record[key]
-            memo
+        row = @fields.map do |key|
+          record[key]
         end
         CSV.generate_line(row, col_sep: @delimiter,
                           force_quotes: @force_quotes)

data/lib/fluent/plugin/formatter_hash.rb

@@ -21,10 +21,7 @@ module Fluent
     class HashFormatter < Formatter
       Plugin.register_formatter('hash', self)
 
-      include HandleTagAndTimeMixin
-      include StructuredFormatMixin
-
-      def format_record(record)
+      def format(tag, time, record)
         "#{record.to_s}\n"
       end
     end

data/lib/fluent/plugin/formatter_json.rb

@@ -21,9 +21,6 @@ module Fluent
     class JSONFormatter < Formatter
       Plugin.register_formatter('json', self)
 
-      include HandleTagAndTimeMixin
-      include StructuredFormatMixin
-
       config_param :json_parser, :string, default: 'oj'
 
       def configure(conf)
@@ -39,7 +36,7 @@ module Fluent
         end
       end
 
-      def format_record(record)
+      def format(tag, time, record)
         "#{@dump_proc.call(record)}\n"
       end
     end

data/lib/fluent/plugin/formatter_ltsv.rb

@@ -22,18 +22,17 @@ module Fluent
       Plugin.register_formatter('ltsv', self)
 
       # http://ltsv.org/
-      include HandleTagAndTimeMixin
 
       config_param :delimiter, :string, default: "\t"
       config_param :label_delimiter, :string, default: ":"
 
       # TODO: escaping for \t in values
       def format(tag, time, record)
-        filter_record(tag, time, record)
-        formatted = record.inject('') { |result, pair|
-          result << @delimiter if result.length.nonzero?
-          result << "#{pair.first}#{@label_delimiter}#{pair.last}"
-        }
+        formatted = ""
+        record.each do |label, value|
+          formatted << @delimiter if formatted.length.nonzero?
+          formatted << "#{label}#{@label_delimiter}#{value}"
+        end
         formatted << "\n"
         formatted
       end

data/lib/fluent/plugin/formatter_msgpack.rb

@@ -21,10 +21,7 @@ module Fluent
     class MessagePackFormatter < Formatter
       Plugin.register_formatter('msgpack', self)
 
-      include HandleTagAndTimeMixin
-      include StructuredFormatMixin
-
-      def format_record(record)
+      def format(tag, time, record)
         record.to_msgpack
       end
     end

data/lib/fluent/plugin/formatter_out_file.rb

@@ -15,14 +15,14 @@
 #
 
 require 'fluent/plugin/formatter'
+require 'fluent/time'
+require 'yajl'
 
 module Fluent
   module Plugin
     class OutFileFormatter < Formatter
       Plugin.register_formatter('out_file', self)
 
-      include HandleTagAndTimeMixin
-
       config_param :output_time, :bool, default: true
       config_param :output_tag, :bool, default: true
       config_param :delimiter, default: "\t" do |val|
@@ -32,9 +32,42 @@ module Fluent
         else "\t"
         end
       end
+      config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :string
+      config_param :time_format, :string, default: nil
+      config_param :localtime, :bool, default: true # if localtime is false and timezone is nil, then utc
+      config_param :timezone, :string, default: nil
+
+      def configure(conf)
+        # TODO: make a utility method in TimeFormatter to handle these conversion
+        #       copies of this code: plugin_helper/compat_parameters, compat/formatter_utils and here
+        if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
+          conf['time_type'] = 'unixtime'
+        end
+        if conf.has_key?('localtime') || conf.has_key?('utc')
+          if conf.has_key?('localtime') && conf.has_key?('utc')
+            raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
+          elsif conf.has_key?('localtime')
+            conf['localtime'] = Fluent::Config.bool_value(conf['localtime'])
+          elsif conf.has_key?('utc')
+            conf['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
+            # Specifying "localtime false" means using UTC in TimeFormatter
+            # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
+            # There are difference between "Z" and "+0000" in timezone formatting.
+            # TODO: add kwargs to TimeFormatter to specify "using localtime", "using UTC" or "using specified timezone" in more explicit way
+          end
+        end
+
+        super
+
+        @timef = case @time_type
+                 when :float then ->(time){ time.to_r.to_f }
+                 when :unixtime then ->(time){ time.to_i }
+                 else
+                   Fluent::TimeFormatter.new(@time_format, @localtime, @timezone)
+                 end
+      end
 
       def format(tag, time, record)
-        filter_record(tag, time, record)
         header = ''
         header << "#{@timef.format(time)}#{@delimiter}" if @output_time
         header << "#{tag}#{@delimiter}" if @output_tag

data/lib/fluent/plugin/formatter_stdout.rb

@@ -26,14 +26,49 @@ module Fluent
       def configure(conf)
         super
 
-        @sub_formatter = Plugin.new_formatter(@output_type)
+        @sub_formatter = Plugin.new_formatter(@output_type, parent: self.owner)
         @sub_formatter.configure(conf)
       end
 
+      def start
+        super
+        @sub_formatter.start
+      end
+
       def format(tag, time, record)
         header = "#{Time.now.localtime} #{tag}: "
         "#{header}#{@sub_formatter.format(tag, time, record)}"
       end
+
+      def stop
+        @sub_formatter.stop
+        super
+      end
+
+      def before_shutdown
+        @sub_formatter.before_shutdown
+        super
+      end
+
+      def shutdown
+        @sub_formatter.shutdown
+        super
+      end
+
+      def after_shutdown
+        @sub_formatter.after_shutdown
+        super
+      end
+
+      def close
+        @sub_formatter.close
+        super
+      end
+
+      def terminate
+        @sub_formatter.terminate
+        super
+      end
     end
   end
 end

data/lib/fluent/plugin/in_dummy.rb

@@ -26,6 +26,7 @@ module Fluent::Plugin
     helpers :thread, :storage
 
     BIN_NUM = 10
+    DEFAULT_STORAGE_TYPE = 'local'
 
     desc "The value is the tag assigned to the generated events."
     config_param :tag, :string
@@ -33,6 +34,8 @@ module Fluent::Plugin
     config_param :rate, :integer, default: 1
     desc "If specified, each generated event has an auto-incremented key field."
     config_param :auto_increment_key, :string, default: nil
+    desc "The boolean to suspend-and-resume incremental value after restart"
+    config_param :suspend, :bool, default: false
     desc "The dummy data to be generated. An array of JSON hashes or a single JSON hash."
     config_param :dummy, default: [{"message"=>"dummy"}] do |val|
       begin
@@ -58,12 +61,16 @@ module Fluent::Plugin
     def configure(conf)
       super
       @dummy_index = 0
+      config = conf.elements.select{|e| e.name == 'storage' }.first
+      @storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
     end
 
     def start
       super
 
-      @storage = storage_create(type: 'local')
+      @storage.put(:increment_value, 0) unless @storage.get(:increment_value)
+      @storage.put(:dummy_index, 0) unless @storage.get(:dummy_index)
+
       if @auto_increment_key && !@storage.get(:auto_increment_value)
         @storage.put(:auto_increment_value, -1)
       end
@@ -100,7 +107,7 @@ module Fluent::Plugin
       d = @dummy[@dummy_index]
       unless d
         @dummy_index = 0
-        d = @dummy[0]
+        d = @dummy[@dummy_index]
       end
       @dummy_index += 1
       if @auto_increment_key

data/lib/fluent/plugin/in_exec.rb

@@ -17,14 +17,16 @@
 require 'strptime'
 require 'yajl'
 
-require 'fluent/input'
+require 'fluent/plugin/input'
 require 'fluent/time'
 require 'fluent/timezone'
 require 'fluent/config/error'
 
-module Fluent
-  class ExecInput < Input
-    Plugin.register_input('exec', self)
+module Fluent::Plugin
+  class ExecInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('exec', self)
+
+    helpers :child_process
 
     def initialize
       super
@@ -65,7 +67,7 @@ module Fluent
       end
 
       if !@tag && !@tag_key
-        raise ConfigError, "'tag' or 'tag_key' option is required on exec input"
+        raise Fleunt::ConfigError, "'tag' or 'tag_key' option is required on exec input"
       end
 
       if @time_key
@@ -90,15 +92,15 @@ module Fluent
       case @format
       when 'tsv'
         if @keys.empty?
-          raise ConfigError, "keys option is required on exec input for tsv format"
+          raise Fluent::ConfigError, "keys option is required on exec input for tsv format"
         end
-        ExecUtil::TSVParser.new(@keys, method(:on_message))
+        Fluent::ExecUtil::TSVParser.new(@keys, method(:on_message))
       when 'json'
-        ExecUtil::JSONParser.new(method(:on_message))
+        Fluent::ExecUtil::JSONParser.new(method(:on_message))
       when 'msgpack'
-        ExecUtil::MessagePackParser.new(method(:on_message))
+        Fluent::ExecUtil::MessagePackParser.new(method(:on_message))
       else
-        ExecUtil::TextParserWrapperParser.new(conf, method(:on_message))
+        Fluent::ExecUtil::TextParserWrapperParser.new(conf, method(:on_message))
       end
     end
 
@@ -106,57 +108,18 @@ module Fluent
       super
 
       if @run_interval
-        @finished = false
-        @thread = Thread.new(&method(:run_periodic))
-      else
-        @io = IO.popen(@command, "r")
-        @pid = @io.pid
-        @thread = Thread.new(&method(:run))
-      end
-    end
-
-    def shutdown
-      if @run_interval
-        @finished = true
-        # call Thread#run which interupts sleep in order to stop run_periodic thread immediately.
-        @thread.run
-        @thread.join
-      else
-        begin
-          Process.kill(:TERM, @pid)
-        rescue #Errno::ECHILD, Errno::ESRCH, Errno::EPERM
-        end
-        if @thread.join(60)  # TODO wait time
-          return
+        child_process_execute(:exec_input, @command, interval: @run_interval, mode: [:read]) do |io|
+          run(io)
         end
-
-        begin
-          Process.kill(:KILL, @pid)
-        rescue #Errno::ECHILD, Errno::ESRCH, Errno::EPERM
+      else
+        child_process_execute(:exec_input, @command, immediate: true, mode: [:read]) do |io|
+          run(io)
         end
-        @thread.join
       end
-
-      super
     end
 
-    def run
-      @parser.call(@io)
-    end
-
-    def run_periodic
-      sleep @run_interval
-      until @finished
-        begin
-          io = IO.popen(@command, "r")
-          @parser.call(io)
-          Process.waitpid(io.pid)
-          sleep @run_interval
-        rescue
-          log.error "exec failed to run or shutdown child process", error: $!
-          log.warn_backtrace $!.backtrace
-        end
-      end
+    def run(io)
+      @parser.call(io)
     end
 
     private
@@ -174,7 +137,7 @@ module Fluent
         if val = record.delete(@time_key)
           time = @time_parse_proc.call(val)
         else
-          time = Engine.now
+          time = Fluent::EventTime.now
         end
       end
 

data/lib/fluent/plugin/in_forward.rb

@@ -169,10 +169,11 @@ module Fluent
 
       if entries.class == String
         # PackedForward
-        es = MessagePackEventStream.new(entries)
+        option = msg[2]
+        size = (option && option['size']) || 0
+        es = MessagePackEventStream.new(entries, nil, size.to_i)
         es = check_and_skip_invalid_event(tag, es, source) if @skip_invalid_event
         router.emit_stream(tag, es)
-        option = msg[2]
 
       elsif entries.class == Array
         # Forward
@@ -265,7 +266,7 @@ module Fluent
           @y = Yajl::Parser.new
           @y.on_parse_complete = lambda { |obj|
             option = @on_message.call(obj, @chunk_counter, @source)
-            respond option if option
+            respond option
             @chunk_counter = 0
           }
         else

data/lib/fluent/plugin/in_object_space.rb

@@ -17,11 +17,13 @@
 require 'cool.io'
 require 'yajl'
 
-require 'fluent/input'
+require 'fluent/plugin/input'
 
-module Fluent
-  class ObjectSpaceInput < Input
-    Plugin.register_input('object_space', self)
+module Fluent::Plugin
+  class ObjectSpaceInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('object_space', self)
+
+    helpers :timer
 
     def initialize
       super
@@ -31,48 +33,10 @@ module Fluent
     config_param :tag, :string
     config_param :top, :integer, default: 15
 
-    class TimerWatcher < Coolio::TimerWatcher
-      def initialize(interval, repeat, log, &callback)
-        @callback = callback
-        @log = log
-        super(interval, repeat)
-      end
-
-      def on_timer
-        @callback.call
-      rescue
-        # TODO log?
-        @log.error $!.to_s
-        @log.error_backtrace
-      end
-    end
-
-    def configure(conf)
-      super
-    end
-
     def start
       super
 
-      @loop = Coolio::Loop.new
-      @timer = TimerWatcher.new(@emit_interval, true, log, &method(:on_timer))
-      @loop.attach(@timer)
-      @thread = Thread.new(&method(:run))
-    end
-
-    def shutdown
-      @loop.watchers.each {|w| w.detach }
-      @loop.stop
-      @thread.join
-
-      super
-    end
-
-    def run
-      @loop.run
-    rescue
-      log.error "unexpected error", error: $!.to_s
-      log.error_backtrace
+      timer_execute(:object_space_input, @emit_interval, &method(:on_timer))
     end
 
     class Counter
@@ -93,7 +57,7 @@ module Fluent
     end
 
     def on_timer
-      now = Engine.now
+      now = Fluent::EventTime.now
 
       array = []
       map = {}