fluentd 0.12.40 → 1.6.2

data/lib/fluent/plugin/out_exec_filter.rb

@@ -13,142 +13,143 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
-
-require 'yajl'
-
-require 'fluent/output'
+require 'fluent/plugin/output'
 require 'fluent/env'
-require 'fluent/time'
-require 'fluent/timezone'
-require 'fluent/plugin/exec_util'
 require 'fluent/config/error'
 
-module Fluent
-  class ExecFilterOutput < BufferedOutput
-    Plugin.register_output('exec_filter', self)
+require 'yajl'
 
-    def initialize
-      super
-      require 'fluent/timezone'
-    end
+module Fluent::Plugin
+  class ExecFilterOutput < Output
+    Fluent::Plugin.register_output('exec_filter', self)
+
+    helpers :compat_parameters, :inject, :formatter, :parser, :extract, :child_process, :event_emitter
 
     desc 'The command (program) to execute.'
     config_param :command, :string
 
-    config_param :remove_prefix, :string, default: nil
-    config_param :add_prefix, :string, default: nil
+    config_param :remove_prefix, :string, default: nil, deprecated: "use @label instead for event routing"
+    config_param :add_prefix, :string, default: nil, deprecated: "use @label instead for event routing"
+
+    config_section :inject do
+      config_set_default :time_type, :unixtime
+    end
 
-    desc "The format used to map the incoming event to the program input.(#{ExecUtil::SUPPORTED_FORMAT.keys.join(',')})"
-    config_param :in_format, default: :tsv do |val|
-      f = ExecUtil::SUPPORTED_FORMAT[val]
-      raise ConfigError, "Unsupported in_format '#{val}'" unless f
-      f
+    config_section :format do
+      config_set_default :@type, 'tsv'
+      config_set_default :localtime, true
     end
-    desc 'Specify comma-separated values for tsv format.'
-    config_param :in_keys, default: [] do |val|
-      val.split(',')
+
+    config_section :parse do
+      config_set_default :@type, 'tsv'
+      config_set_default :time_key, nil
+      config_set_default :time_format, nil
+      config_set_default :localtime, true
+      config_set_default :estimate_current_event, false
     end
-    desc 'The name of the key to use as the event tag.'
-    config_param :in_tag_key, default: nil
-    desc 'The name of the key to use as the event time.'
-    config_param :in_time_key, default: nil
-    desc 'The format for event time used when the in_time_key parameter is specified.(Defauls is UNIX time)'
-    config_param :in_time_format, default: nil
-
-    desc "The format used to process the program output.(#{ExecUtil::SUPPORTED_FORMAT.keys.join(',')})"
-    config_param :out_format, default: :tsv do |val|
-      f = ExecUtil::SUPPORTED_FORMAT[val]
-      raise ConfigError, "Unsupported out_format '#{val}'" unless f
-      f
+
+    config_section :extract do
+      config_set_default :time_type, :float
     end
-    desc 'Specify comma-separated values for tsv format.'
-    config_param :out_keys, default: [] do |val|  # for tsv format
-      val.split(',')
+
+    config_section :buffer do
+      config_set_default :flush_mode, :interval
+      config_set_default :flush_interval, 1
     end
-    desc 'The name of the key to use as the event tag.'
-    config_param :out_tag_key, default: nil
-    desc 'The name of the key to use as the event time.'
-    config_param :out_time_key, default: nil
-    desc 'The format for event time used when the in_time_key parameter is specified.(Defauls is UNIX time)'
-    config_param :out_time_format, default: nil
 
     config_param :tag, :string, default: nil
 
-    config_param :time_key, :string, default: nil
-    config_param :time_format, :string, default: nil
+    config_param :tag_key, :string, default: nil, deprecated: "use 'tag_key' in <inject>/<extract> instead"
+    config_param :time_key, :string, default: nil, deprecated: "use 'time_key' in <inject>/<extract> instead"
+    config_param :time_format, :string, default: nil, deprecated: "use 'time_format' in <inject>/<extract> instead"
+
+    desc 'The default block size to read if parser requires partial read.'
+    config_param :read_block_size, :size, default: 10240 # 10k
 
-    desc 'If true, use localtime with in_time_format.'
-    config_param :localtime, :bool, default: true
-    desc 'If true, use timezone with in_time_format.'
-    config_param :timezone, :string, default: nil
     desc 'The number of spawned process for command.'
     config_param :num_children, :integer, default: 1
 
-    desc 'Respawn command when command exit.'
+    desc 'Respawn command when command exit. ["none", "inf" or positive integer for times to respawn (default: none)]'
     # nil, 'none' or 0: no respawn, 'inf' or -1: infinite times, positive integer: try to respawn specified times only
     config_param :child_respawn, :string, default: nil
 
     # 0: output logs for all of messages to emit
     config_param :suppress_error_log_interval, :time, default: 0
 
-    config_set_default :flush_interval, 1
-
-    def configure(conf)
-      if tag_key = conf['tag_key']
-        # TODO obsoleted?
-        @in_tag_key = tag_key
-        @out_tag_key = tag_key
-      end
-
-      if time_key = conf['time_key']
-        # TODO obsoleted?
-        @in_time_key = time_key
-        @out_time_key = time_key
+    attr_reader :formatter, :parser # for tests
+
+    KEYS_FOR_IN_AND_OUT = {
+      'tag_key' => ['in_tag_key', 'out_tag_key'],
+      'time_key' => ['in_time_key', 'out_time_key'],
+      'time_format' => ['in_time_format', 'out_time_format'],
+    }
+    COMPAT_INJECT_PARAMS = {
+      'in_tag_key' => 'tag_key',
+      'in_time_key' => 'time_key',
+      'in_time_format' => 'time_format',
+    }
+    COMPAT_FORMAT_PARAMS = {
+      'in_format' => '@type',
+      'in_keys' => 'keys',
+    }
+    COMPAT_PARSE_PARAMS = {
+      'out_format' => '@type',
+      'out_keys' => 'keys',
+      'out_stream_buffer_size' => 'stream_buffer_size',
+    }
+    COMPAT_EXTRACT_PARAMS = {
+      'out_tag_key' => 'tag_key',
+      'out_time_key' => 'time_key',
+      'out_time_format' => 'time_format',
+    }
+
+    def exec_filter_compat_parameters_copy_to_subsection!(conf, subsection_name, params)
+      return unless conf.elements(subsection_name).empty?
+      return unless params.keys.any?{|k| conf.has_key?(k) }
+      hash = {}
+      params.each_pair do |compat, current|
+        hash[current] = conf[compat] if conf.has_key?(compat)
       end
+      conf.elements << Fluent::Config::Element.new(subsection_name, '', hash, [])
+    end
 
-      if time_format = conf['time_format']
-        # TODO obsoleted?
-        @in_time_format = time_format
-        @out_time_format = time_format
+    def exec_filter_compat_parameters_convert!(conf)
+      KEYS_FOR_IN_AND_OUT.each_pair do |inout, keys|
+        if conf.has_key?(inout)
+          keys.each do |k|
+            conf[k] = conf[inout]
+          end
+        end
       end
+      exec_filter_compat_parameters_copy_to_subsection!(conf, 'inject', COMPAT_INJECT_PARAMS)
+      exec_filter_compat_parameters_copy_to_subsection!(conf, 'format', COMPAT_FORMAT_PARAMS)
+      exec_filter_compat_parameters_copy_to_subsection!(conf, 'parse', COMPAT_PARSE_PARAMS)
+      exec_filter_compat_parameters_copy_to_subsection!(conf, 'extract', COMPAT_EXTRACT_PARAMS)
+    end
 
-      super
+    def configure(conf)
+      exec_filter_compat_parameters_convert!(conf)
+      compat_parameters_convert(conf, :buffer)
 
-      if localtime = conf['localtime']
-        @localtime = true
-      elsif utc = conf['utc']
-        @localtime = false
+      if inject_section = conf.elements('inject').first
+        if inject_section.has_key?('time_format')
+          inject_section['time_type'] ||= 'string'
+        end
       end
-
-      if conf['timezone']
-        @timezone = conf['timezone']
-        Fluent::Timezone.validate!(@timezone)
+      if extract_section = conf.elements('extract').first
+        if extract_section.has_key?('time_format')
+          extract_section['time_type'] ||= 'string'
+        end
       end
 
-      if !@tag && !@out_tag_key
-        raise ConfigError, "'tag' or 'out_tag_key' option is required on exec_filter output"
-      end
+      super
 
-      if @in_time_key
-        if f = @in_time_format
-          tf = TimeFormatter.new(f, @localtime, @timezone)
-          @time_format_proc = tf.method(:format)
-        else
-          @time_format_proc = Proc.new {|time| time.to_s }
-        end
-      elsif @in_time_format
-        log.warn "in_time_format effects nothing when in_time_key is not specified: #{conf}"
+      if !@tag && (!@extract_config || !@extract_config.tag_key)
+        raise Fluent::ConfigError, "'tag' or '<extract> tag_key </extract>' option is required on exec_filter output"
       end
 
-      if @out_time_key
-        if f = @out_time_format
-          @time_parse_proc = Proc.new {|str| Time.strptime(str, f).to_i }
-        else
-          @time_parse_proc = Proc.new {|str| str.to_i }
-        end
-      elsif @out_time_format
-        log.warn "out_time_format effects nothing when out_time_key is not specified: #{conf}"
-      end
+      @formatter = formatter_create
+      @parser = parser_create
 
       if @remove_prefix
         @removed_prefix_string = @remove_prefix + '.'
@@ -158,30 +159,6 @@ module Fluent
         @added_prefix_string = @add_prefix + '.'
       end
 
-      case @in_format
-      when :tsv
-        if @in_keys.empty?
-          raise ConfigError, "in_keys option is required on exec_filter output for tsv in_format"
-        end
-        @formatter = ExecUtil::TSVFormatter.new(@in_keys)
-      when :json
-        @formatter = ExecUtil::JSONFormatter.new
-      when :msgpack
-        @formatter = ExecUtil::MessagePackFormatter.new
-      end
-
-      case @out_format
-      when :tsv
-        if @out_keys.empty?
-          raise ConfigError, "out_keys option is required on exec_filter output for tsv in_format"
-        end
-        @parser = ExecUtil::TSVParser.new(@out_keys, method(:on_message))
-      when :json
-        @parser = ExecUtil::JSONParser.new(method(:on_message))
-      when :msgpack
-        @parser = ExecUtil::MessagePackParser.new(method(:on_message))
-      end
-
       @respawns = if @child_respawn.nil? or @child_respawn == 'none' or @child_respawn == '0'
                     0
                   elsif @child_respawn == 'inf' or @child_respawn == '-1'
@@ -196,190 +173,146 @@ module Fluent
       @next_log_time = Time.now.to_i
     end
 
+    def multi_workers_ready?
+      true
+    end
+
+    ExecutedProcess = Struct.new(:mutex, :pid, :respawns, :readio, :writeio)
+
     def start
       super
 
+      @children_mutex = Mutex.new
       @children = []
       @rr = 0
-      begin
-        @num_children.times do
-          c = ChildProcess.new(@parser, @respawns, log)
-          c.start(@command)
-          @children << c
-        end
-      rescue
-        shutdown
-        raise
-      end
-    end
-
-    def before_shutdown
-      super
-      log.debug "out_exec_filter#before_shutdown called"
-      @children.each {|c|
-        c.finished = true
-      }
-      sleep 0.5  # TODO wait time before killing child process
-    end
 
-    def shutdown
-      super
-
-      @children.reject! {|c|
-        c.shutdown
-        true
+      exit_callback = ->(status){
+        c = @children.select{|child| child.pid == status.pid }.first
+        if c
+          unless self.stopped?
+            log.warn "child process exits with error code", code: status.to_i, status: status.exitstatus, signal: status.termsig
+          end
+          c.mutex.synchronize do
+            (c.writeio && c.writeio.close) rescue nil
+            (c.readio && c.readio.close) rescue nil
+            c.pid = c.readio = c.writeio = nil
+          end
+        end
       }
-    end
-
-    def format_stream(tag, es)
-      if @remove_prefix
-        if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or tag == @removed_prefix
-          tag = tag[@removed_length..-1] || ''
+      child_process_callback = ->(index, readio, writeio){
+        pid = child_process_id
+        c = @children[index]
+        writeio.sync = true
+        c.mutex.synchronize do
+          c.pid = pid
+          c.respawns = @respawns
+          c.readio = readio
+          c.writeio = writeio
         end
-      end
 
-      out = ''
-
-      es.each {|time,record|
-        if @in_time_key
-          record[@in_time_key] = @time_format_proc.call(time)
-        end
-        if @in_tag_key
-          record[@in_tag_key] = tag
+        run(readio)
+      }
+      execute_child_process = ->(index){
+        child_process_execute("out_exec_filter_child#{index}".to_sym, @command, on_exit_callback: exit_callback) do |readio, writeio|
+          child_process_callback.call(index, readio, writeio)
         end
-        @formatter.call(record, out)
       }
 
-      out
-    end
-
-    def write(chunk)
-      r = @rr = (@rr + 1) % @children.length
-      @children[r].write chunk
-    end
-
-    class ChildProcess
-      attr_accessor :finished
-
-      def initialize(parser, respawns=0, log = $log)
-        @pid = nil
-        @thread = nil
-        @parser = parser
-        @respawns = respawns
-        @mutex = Mutex.new
-        @finished = nil
-        @log = log
-      end
-
-      def start(command)
-        @command = command
-        @mutex.synchronize do
-          @io = IO.popen(command, "r+")
-          @pid = @io.pid
-          @io.sync = true
-          @thread = Thread.new(&method(:run))
+      @children_mutex.synchronize do
+        @num_children.times do |i|
+          @children << ExecutedProcess.new(Mutex.new, nil, 0, nil, nil)
+          execute_child_process.call(i)
         end
-        @finished = false
       end
 
-      def kill_child(join_wait)
-        begin
-          Process.kill(:TERM, @pid)
-        rescue #Errno::ECHILD, Errno::ESRCH, Errno::EPERM
-          # Errno::ESRCH 'No such process', ignore
-          # child process killed by signal chained from fluentd process
-        end
-        if @thread.join(join_wait)
-          # @thread successfully shutdown
-          return
-        end
-        begin
-          Process.kill(:KILL, @pid)
-        rescue #Errno::ECHILD, Errno::ESRCH, Errno::EPERM
-          # ignore if successfully killed by :TERM
+      if @respawns != 0
+        thread_create(:out_exec_filter_respawn_monitor) do
+          while thread_current_running?
+            @children.each_with_index do |c, i|
+              if c.mutex && c.mutex.synchronize{ c.pid.nil? && c.respawns != 0 }
+                respawns = c.mutex.synchronize do
+                  c.respawns -= 1 if c.respawns > 0
+                  c.respawns
+                end
+                log.info "respawning child process", num: i, respawn_counter: respawns
+                execute_child_process.call(i)
+              end
+            end
+            sleep 0.2
+          end
         end
-        @thread.join
       end
+    end
 
-      def shutdown
-        @finished = true
-        @mutex.synchronize do
-          kill_child(60) # TODO wait time
-        end
-      end
+    def terminate
+      @children = []
+      super
+    end
 
-      def write(chunk)
-        begin
-          chunk.write_to(@io)
-        rescue Errno::EPIPE => e
-          # Broken pipe (child process unexpectedly exited)
-          @log.warn "exec_filter Broken pipe, child process maybe exited.", command: @command
-          if try_respawn
-            retry # retry chunk#write_to with child respawned
-          else
-            raise e # to retry #write with other ChildProcess instance (when num_children > 1)
-          end
+    def tag_remove_prefix(tag)
+      if @remove_prefix
+        if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or tag == @removed_prefix_string
+          tag = tag[@removed_length..-1] || ''
         end
       end
+      tag
+    end
 
-      def try_respawn
-        return false if @respawns == 0
-        @mutex.synchronize do
-          return false if @respawns == 0
-
-          kill_child(5) # TODO wait time
-
-          @io = IO.popen(@command, "r+")
-          @pid = @io.pid
-          @io.sync = true
-          @thread = Thread.new(&method(:run))
+    NEWLINE = "\n"
 
-          @respawns -= 1 if @respawns > 0
-        end
-        @log.warn "exec_filter child process successfully respawned.", command: @command, respawns: @respawns
-        true
+    def format(tag, time, record)
+      tag = tag_remove_prefix(tag)
+      record = inject_values_to_record(tag, time, record)
+      if @formatter.formatter_type == :text_per_line
+        @formatter.format(tag, time, record).chomp + NEWLINE
+      else
+        @formatter.format(tag, time, record)
       end
+    end
 
-      def run
-        @parser.call(@io)
-      rescue
-        @log.error "exec_filter thread unexpectedly failed with an error.", command: @command, error: $!.to_s
-        @log.warn_backtrace $!.backtrace
-      ensure
-        pid, stat = Process.waitpid2(@pid)
-        unless @finished
-          @log.error "exec_filter process unexpectedly exited.", command: @command, ecode: stat.to_i
-          unless @respawns == 0
-            @log.warn "exec_filter child process will respawn for next input data (respawns #{@respawns})."
-          end
+    def write(chunk)
+      try_times = 0
+      while true
+        r = @rr = (@rr + 1) % @children.length
+        if @children[r].pid && writeio = @children[r].writeio
+          chunk.write_to(writeio)
+          break
         end
+        try_times += 1
+        raise "no healthy child processes exist" if try_times >= @children.length
       end
     end
 
-    def on_message(record)
-      if val = record.delete(@out_time_key)
-        time = @time_parse_proc.call(val)
-      else
-        time = Engine.now
-      end
-
-      if val = record.delete(@out_tag_key)
-        tag = if @add_prefix
-                @added_prefix_string + val
-              else
-                val
-              end
+    def run(io)
+      case
+      when @parser.implement?(:parse_io)
+        @parser.parse_io(io, &method(:on_record))
+      when @parser.implement?(:parse_partial_data)
+        until io.eof?
+          @parser.parse_partial_data(io.readpartial(@read_block_size), &method(:on_record))
+        end
+      when @parser.parser_type == :text_per_line
+        io.each_line do |line|
+          @parser.parse(line.chomp, &method(:on_record))
+        end
       else
-        tag = @tag
+        @parser.parse(io.read, &method(:on_record))
       end
+    end
 
+    def on_record(time, record)
+      tag = extract_tag_from_record(record)
+      tag = @added_prefix_string + tag if tag && @add_prefix
+      tag ||= @tag
+      time ||= extract_time_from_record(record) || Fluent::EventTime.now
       router.emit(tag, time, record)
-    rescue
+    rescue => e
       if @suppress_error_log_interval == 0 || Time.now.to_i > @next_log_time
-        log.error "exec_filter failed to emit", error: $!.to_s, error_class: $!.class.to_s, record: Yajl.dump(record)
-        log.warn_backtrace $!.backtrace
+        log.error "exec_filter failed to emit", record: Yajl.dump(record), error: e
+        log.error_backtrace e.backtrace
         @next_log_time = Time.now.to_i + @suppress_error_log_interval
       end
+      router.emit_error_event(tag, time, record, e) if tag && time && record
     end
   end
 end