fluentd 0.14.7-x64-mingw32 → 0.14.10-x64-mingw32

data/lib/fluent/plugin/parser_csv.rb

@@ -20,13 +20,19 @@ require 'csv'
 
 module Fluent
   module Plugin
-    class CSVParser < ValuesParser
+    class CSVParser < Parser
       Plugin.register_parser('csv', self)
 
+      desc 'Names of fields included in each lines'
+      config_param :keys, :array, value_type: :string
+      desc 'The delimiter character (or string) of CSV values'
       config_param :delimiter, :string, default: ','
 
-      def parse(text)
-        yield values_map(CSV.parse_line(text, col_sep: @delimiter))
+      def parse(text, &block)
+        values = CSV.parse_line(text, col_sep: @delimiter)
+        r = Hash[@keys.zip(values)]
+        time, record = convert_values(parse_time(r), r)
+        yield time, record
       end
     end
   end

data/lib/fluent/plugin/parser_json.rb

@@ -19,61 +19,63 @@ require 'fluent/env'
 require 'fluent/time'
 
 require 'yajl'
+require 'json'
 
 module Fluent
   module Plugin
     class JSONParser < Parser
       Plugin.register_parser('json', self)
 
-      config_param :time_key, :string, default: 'time'
-      config_param :json_parser, :string, default: 'oj'
+      config_set_default :time_key, 'time'
+      config_param :json_parser, :enum, list: [:oj, :yajl, :json], default: :oj
 
-      def configure(conf)
-        super
+      config_set_default :time_type, :float
 
-        if @time_format
-          @time_parser = time_parser_create
-          @mutex = Mutex.new
+      def configure(conf)
+        if conf.has_key?('time_format')
+          conf['time_type'] ||= 'string'
         end
 
-        begin
-          raise LoadError unless @json_parser == 'oj'
+        super
+        @load_proc, @error_class = configure_json_parser(@json_parser)
+      end
+
+      def configure_json_parser(name)
+        case name
+        when :oj
           require 'oj'
           Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
-          @load_proc = Oj.method(:load)
-          @error_class = Oj::ParseError
-        rescue LoadError
-          @load_proc = Yajl.method(:load)
-          @error_class = Yajl::ParseError
+          [Oj.method(:load), Oj::ParseError]
+        when :json then [JSON.method(:load), JSON::ParserError]
+        when :yajl then [Yajl.method(:load), Yajl::ParseError]
+        else
+          raise "BUG: unknown json parser specified: #{name}"
         end
+      rescue LoadError
+        name = :yajl
+        log.info "Oj is not installed, and failing back to Yajl for json parser" if log
+        retry
       end
 
       def parse(text)
-        record = @load_proc.call(text)
-
-        value = @keep_time_key ? record[@time_key] : record.delete(@time_key)
-        if value
-          if @time_format
-            time = @mutex.synchronize { @time_parser.parse(value) }
-          else
-            begin
-              time = Fluent::EventTime.from_time(Time.at(value.to_f))
-            rescue => e
-              raise ParserError, "invalid time value: value = #{value}, error_class = #{e.class.name}, error = #{e.message}"
-            end
-          end
-        else
-          if @estimate_current_event
-            time = Fluent::EventTime.now
-          else
-            time = nil
-          end
-        end
-
+        r = @load_proc.call(text)
+        time, record = convert_values(parse_time(r), r)
         yield time, record
       rescue @error_class
         yield nil, nil
       end
+
+      def parser_type
+        :text
+      end
+
+      def parse_io(io, &block)
+        y = Yajl::Parser.new
+        y.on_parse_complete = ->(record){
+          block.call(parse_time(record), record)
+        }
+        y.parse(io)
+      end
     end
   end
 end

data/lib/fluent/plugin/parser_ltsv.rb

@@ -15,35 +15,27 @@
 #
 
 require 'fluent/plugin/parser'
-require 'fluent/time'
 
 module Fluent
   module Plugin
-    class LabeledTSVParser < ValuesParser
+    class LabeledTSVParser < Parser
       Plugin.register_parser('ltsv', self)
 
-      config_param :delimiter,       :string, default: "\t"
+      desc 'The delimiter character (or string) of TSV values'
+      config_param :delimiter, :string, default: "\t"
+      desc 'The delimiter character between field name and value'
       config_param :label_delimiter, :string, default: ":"
-      config_param :time_key, :string, default: "time"
 
-      def configure(conf)
-        # this assignment is not to raise ConfigError in ValuesParser#configure
-        conf['keys'] = conf['time_key'] || 'time'
-        super(conf)
-      end
+      config_set_default :time_key, 'time'
 
       def parse(text)
-        # TODO: thread unsafe: @keys might be changed by other threads
-        @keys  = []
-        values = []
-
-        text.split(delimiter).each do |pair|
-          key, value = pair.split(label_delimiter, 2)
-          @keys.push(key)
-          values.push(value)
+        r = {}
+        text.split(@delimiter).each do |pair|
+          key, value = pair.split(@label_delimiter, 2)
+          r[key] = value
         end
-
-        yield values_map(values)
+        time, record = convert_values(parse_time(r), r)
+        yield time, record
       end
     end
   end

data/lib/fluent/plugin/parser_msgpack.rb

@@ -0,0 +1,50 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+require 'fluent/msgpack_factory'
+
+module Fluent
+  module Plugin
+    class MessagePackParser < Parser
+      Plugin.register_parser('msgpack', self)
+
+      def configure(conf)
+        super
+        @unpacker = Fluent::MessagePackFactory.engine_factory.unpacker
+      end
+
+      def parser_type
+        :binary
+      end
+
+      def parse(data)
+        @unpacker.feed_each(data) do |obj|
+          yield convert_values(parse_time(obj), obj)
+        end
+      end
+      alias parse_partial_data parse
+
+      def parse_io(io, &block)
+        u = Fluent::MessagePackFactory.engine_factory.unpacker(io)
+        u.each do |obj|
+          time, record = convert_values(parse_time(obj), obj)
+          yield time, record
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_regexp.rb

@@ -1,33 +1,26 @@
 module Fluent
   module Plugin
     class RegexpParser < Parser
-      include Fluent::Compat::TypeConverter
-
       Plugin.register_parser("regexp", self)
 
-      config_param :expression, :string, default: ""
+      config_param :expression, :string
       config_param :ignorecase, :bool, default: false
       config_param :multiline, :bool, default: false
-      config_param :time_key, :string, default: 'time'
 
-      def initialize
-        super
-        @mutex = Mutex.new
-      end
+      config_set_default :time_key, 'time'
 
       def configure(conf)
         super
-        @time_parser = time_parser_create
-        unless @expression.empty?
-          if @expression[0] == "/" && @expression[-1] == "/"
-            regexp_option = 0
-            regexp_option |= Regexp::IGNORECASE if @ignorecase
-            regexp_option |= Regexp::MULTILINE if @multiline
-            @regexp = Regexp.new(@expression[1..-2], regexp_option)
-          else
-            raise Fluent::ConfigError, "expression must start with `/` and end with `/`: #{@expression}"
-          end
-        end
+
+        expr = if @expression[0] == "/" && @expression[-1] == "/"
+                 @expression[1..-2]
+               else
+                 @expression
+               end
+        regexp_option = 0
+        regexp_option |= Regexp::IGNORECASE if @ignorecase
+        regexp_option |= Regexp::MULTILINE if @multiline
+        @regexp = Regexp.new(expr, regexp_option)
       end
 
       def parse(text)
@@ -37,34 +30,14 @@ module Fluent
           return
         end
 
-        time = nil
-        record = {}
-
+        r = {}
         m.names.each do |name|
           if value = m[name]
-            if name == @time_key
-              time = @mutex.synchronize { @time_parser.parse(value) }
-              if @keep_time_key
-                record[name] = if @type_converters.nil?
-                                 value
-                               else
-                                 convert_type(name, value)
-                               end
-              end
-            else
-              record[name] = if @type_converters.nil?
-                               value
-                             else
-                               convert_type(name, value)
-                             end
-            end
+            r[name] = value
           end
         end
 
-        if @estimate_current_event
-          time ||= Fluent::EventTime.now
-        end
-
+        time, record = convert_values(parse_time(r), r)
         yield time, record
       end
     end

data/lib/fluent/plugin/parser_tsv.rb

@@ -15,13 +15,15 @@
 #
 
 require 'fluent/plugin/parser'
-require 'fluent/time'
 
 module Fluent
   module Plugin
-    class TSVParser < ValuesParser
+    class TSVParser < Parser
       Plugin.register_parser('tsv', self)
 
+      desc 'Names of fields included in each lines'
+      config_param :keys, :array, value_type: :string
+      desc 'The delimiter character (or string) of TSV values'
       config_param :delimiter, :string, default: "\t"
 
       def configure(conf)
@@ -30,7 +32,10 @@ module Fluent
       end
 
       def parse(text)
-        yield values_map(text.split(@delimiter, @key_num))
+        values = text.split(@delimiter, @key_num)
+        r = Hash[@keys.zip(values)]
+        time, record = convert_values(parse_time(r), r)
+        yield time, record
       end
     end
   end

data/lib/fluent/plugin_helper.rb

@@ -24,6 +24,8 @@ require 'fluent/plugin_helper/parser'
 require 'fluent/plugin_helper/formatter'
 require 'fluent/plugin_helper/inject'
 require 'fluent/plugin_helper/extract'
+require 'fluent/plugin_helper/socket'
+require 'fluent/plugin_helper/server'
 require 'fluent/plugin_helper/retry_state'
 require 'fluent/plugin_helper/compat_parameters'
 
@@ -36,7 +38,14 @@ module Fluent
     end
 
     def helpers(*snake_case_symbols)
-      helper_modules = snake_case_symbols.map{|name| Fluent::PluginHelper.const_get(name.to_s.split('_').map(&:capitalize).join) }
+      helper_modules = []
+      snake_case_symbols.each do |name|
+        begin
+          helper_modules << Fluent::PluginHelper.const_get(name.to_s.split('_').map(&:capitalize).join)
+        rescue NameError
+          raise "Unknown plugin helper:#{name}"
+        end
+      end
       include(*helper_modules)
     end
   end

data/lib/fluent/plugin_helper/child_process.rb

@@ -49,39 +49,60 @@ module Fluent
         ::Thread.current[:_fluentd_plugin_helper_child_process_pid]
       end
 
-      def child_process_exit_status
-        ::Thread.current[:_fluentd_plugin_helper_child_process_exit_status]
+      def child_process_exist?(pid)
+        pinfo = @_child_process_processes[pid]
+        return false unless pinfo
+
+        return false if pinfo.exit_status
+
+        true
       end
 
+      # on_exit_callback = ->(status){ ... }
+      # status is an instance of Process::Status
+      # On Windows, exitstatus=0 and termsig=nil even when child process was killed.
       def child_process_execute(
           title, command,
           arguments: nil, subprocess_name: nil, interval: nil, immediate: false, parallel: false,
           mode: [:read, :write], stderr: :discard, env: {}, unsetenv: false, chdir: nil,
           internal_encoding: 'utf-8', external_encoding: 'ascii-8bit', scrub: true, replace_string: nil,
+          wait_timeout: nil, on_exit_callback: nil,
           &block
       )
         raise ArgumentError, "BUG: title must be a symbol" unless title.is_a? Symbol
         raise ArgumentError, "BUG: arguments required if subprocess name is replaced" if subprocess_name && !arguments
 
+        mode ||= []
+        mode = [] unless block
         raise ArgumentError, "BUG: invalid mode specification" unless mode.all?{|m| MODE_PARAMS.include?(m) }
         raise ArgumentError, "BUG: read_with_stderr is exclusive with :read and :stderr" if mode.include?(:read_with_stderr) && (mode.include?(:read) || mode.include?(:stderr))
         raise ArgumentError, "BUG: invalid stderr handling specification" unless STDERR_OPTIONS.include?(stderr)
 
-        raise ArgumentError, "BUG: block not specified which receive i/o object" unless block_given?
-        raise ArgumentError, "BUG: number of block arguments are different from size of mode" unless block.arity == mode.size
+        raise ArgumentError, "BUG: number of block arguments are different from size of mode" if block && block.arity != mode.size
 
         running = false
         callback = ->(*args) {
           running = true
           begin
-            block.call(*args)
+            block && block.call(*args)
           ensure
             running = false
           end
         }
 
+        retval = nil
+        execute_child_process = ->(){
+          child_process_execute_once(
+            title, command, arguments,
+            subprocess_name, mode, stderr, env, unsetenv, chdir,
+            internal_encoding, external_encoding, scrub, replace_string,
+            wait_timeout, on_exit_callback,
+            &callback
+          )
+        }
+
         if immediate || !interval
-          child_process_execute_once(title, command, arguments, subprocess_name, mode, stderr, env, unsetenv, chdir, internal_encoding, external_encoding, scrub, replace_string, &callback)
+          retval = execute_child_process.call
         end
 
         if interval
@@ -89,10 +110,12 @@ module Fluent
             if !parallel && running
               log.warn "previous child process is still running. skipped.", title: title, command: command, arguments: arguments, interval: interval, parallel: parallel
             else
-              child_process_execute_once(title, command, arguments, subprocess_name, mode, stderr, env, unsetenv, chdir, internal_encoding, external_encoding, scrub, replace_string, &callback)
+              execute_child_process.call
             end
           end
         end
+
+        retval # nil if interval
       end
 
       def initialize
@@ -101,11 +124,8 @@ module Fluent
         @_child_process_exit_timeout = CHILD_PROCESS_DEFAULT_EXIT_TIMEOUT
         @_child_process_kill_timeout = CHILD_PROCESS_DEFAULT_KILL_TIMEOUT
         @_child_process_mutex = Mutex.new
-      end
-
-      def start
-        super
         @_child_process_processes = {} # pid => ProcessInfo
+        @_child_process_clock_id = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
       end
 
       def stop
@@ -115,86 +135,102 @@ module Fluent
             process_info.thread[:_fluentd_plugin_helper_child_process_running] = false
           end
         end
+
+        super
       end
 
       def shutdown
         @_child_process_mutex.synchronize{ @_child_process_processes.keys }.each do |pid|
           process_info = @_child_process_processes[pid]
-          next if !process_info || !process_info.writeio_in_use
-          begin
-            Timeout.timeout(@_child_process_exit_timeout) do
-              process_info.writeio.close
-            end
-          rescue Timeout::Error
-            log.debug "External process #{process_info.title} doesn't exist after STDIN close in timeout #{@_child_process_exit_timeout}sec"
-          end
+          next if !process_info
+          process_info.writeio && process_info.writeio.close rescue nil
+        end
+
+        super
 
+        @_child_process_mutex.synchronize{ @_child_process_processes.keys }.each do |pid|
+          process_info = @_child_process_processes[pid]
+          next if !process_info
           child_process_kill(process_info)
         end
 
-        super
+        exit_wait_timeout = Process.clock_gettime(@_child_process_clock_id) + @_child_process_exit_timeout
+        while Process.clock_gettime(@_child_process_clock_id) < exit_wait_timeout
+          process_exists = false
+          @_child_process_mutex.synchronize{ @_child_process_processes.keys }.each do |pid|
+            unless @_child_process_processes[pid].exit_status
+              process_exists = true
+              break
+            end
+          end
+          if process_exists
+            sleep CHILD_PROCESS_LOOP_CHECK_INTERVAL
+          else
+            break
+          end
+        end
       end
 
       def close
-        while (pids = @_child_process_mutex.synchronize{ @_child_process_processes.keys }).size > 0
+        while true
+          pids = @_child_process_mutex.synchronize{ @_child_process_processes.keys }
+          break if pids.size < 1
+
+          living_process_exist = false
           pids.each do |pid|
             process_info = @_child_process_processes[pid]
-            if !process_info || !process_info.alive
-              @_child_process_mutex.synchronize{ @_child_process_processes.delete(pid) }
-              next
-            end
+            next if !process_info || process_info.exit_status
 
-            process_info.killed_at ||= Time.now # for illegular case (e.g., created after shutdown)
-            next if Time.now < process_info.killed_at + @_child_process_kill_timeout
+            living_process_exist = true
+
+            process_info.killed_at ||= Process.clock_gettime(@_child_process_clock_id) # for illegular case (e.g., created after shutdown)
+            timeout_at = process_info.killed_at + @_child_process_kill_timeout
+            now = Process.clock_gettime(@_child_process_clock_id)
+            next if now < timeout_at
 
             child_process_kill(process_info, force: true)
-            @_child_process_mutex.synchronize{ @_child_process_processes.delete(pid) }
           end
 
+          break if living_process_exist
+
           sleep CHILD_PROCESS_LOOP_CHECK_INTERVAL
         end
 
         super
       end
 
-      def child_process_kill(process_info, force: false)
-        if !process_info || !process_info.alive
-          return
-        end
+      def terminate
+        @_child_process_processes = {}
 
-        process_info.killed_at = Time.now unless force
+        super
+      end
 
-        begin
-          pid, status = Process.waitpid2(process_info.pid, Process::WNOHANG)
-          if pid && status
-            process_info.thread[:_fluentd_plugin_helper_child_process_exit_status] = status
-            process_info.alive = false
-          end
-        rescue Errno::ECHILD, Errno::ESRCH, Errno::EPERM
-          process_info.alive = false
-        rescue
-          # ignore
-        end
-        if !process_info.alive
-          return
-        end
+      def child_process_kill(pinfo, force: false)
+        return if !pinfo
+        pinfo.killed_at = Process.clock_gettime(@_child_process_clock_id) unless force
 
+        pid = pinfo.pid
         begin
-          signal = (Fluent.windows? || force) ? :KILL : :TERM
-          Process.kill(signal, process_info.pid)
-          if force
-            process_info.alive = false
+          if !pinfo.exit_status && child_process_exist?(pid)
+            signal = (Fluent.windows? || force) ? :KILL : :TERM
+            Process.kill(signal, pinfo.pid)
           end
         rescue Errno::ECHILD, Errno::ESRCH
-          process_info.alive = false
+          # ignore
         end
       end
 
-      ProcessInfo = Struct.new(:title, :thread, :pid, :readio, :readio_in_use, :writeio, :writeio_in_use, :stderrio, :stderrio_in_use, :wait_thread, :alive, :killed_at)
+      ProcessInfo = Struct.new(
+        :title,
+        :thread, :pid,
+        :readio, :readio_in_use, :writeio, :writeio_in_use, :stderrio, :stderrio_in_use,
+        :wait_thread, :alive, :killed_at, :exit_status,
+        :on_exit_callback, :on_exit_callback_mutex,
+      )
 
       def child_process_execute_once(
           title, command, arguments, subprocess_name, mode, stderr, env, unsetenv, chdir,
-          internal_encoding, external_encoding, scrub, replace_string, &block
+          internal_encoding, external_encoding, scrub, replace_string, wait_timeout, on_exit_callback, &block
       )
         spawn_args = if arguments || subprocess_name
                        [ env, (subprocess_name ? [command, subprocess_name] : command), *(arguments || []) ]
@@ -227,36 +263,40 @@ module Fluent
           writeio, readio, wait_thread = *Open3.popen2e(*spawn_args, spawn_opts)
         else
           writeio, readio, stderrio, wait_thread = *Open3.popen3(*spawn_args, spawn_opts)
-          if !mode.include?(:stderr) # stderr == :discard
-            stderrio.reopen(IO::NULL)
-          end
         end
 
         if mode.include?(:write)
           writeio.set_encoding(external_encoding, internal_encoding, encoding_options)
           writeio_in_use = true
+        else
+          writeio.reopen(IO::NULL) if writeio
         end
         if mode.include?(:read) || mode.include?(:read_with_stderr)
           readio.set_encoding(external_encoding, internal_encoding, encoding_options)
           readio_in_use = true
+        else
+          readio.reopen(IO::NULL) if readio
         end
         if mode.include?(:stderr)
           stderrio.set_encoding(external_encoding, internal_encoding, encoding_options)
           stderrio_in_use = true
+        else
+          stderrio.reopen(IO::NULL) if stderrio && stderrio == :discard
         end
 
         pid = wait_thread.pid # wait_thread => Process::Waiter
 
         io_objects = []
         mode.each do |m|
-          io_objects << case m
-                        when :read then readio
-                        when :write then writeio
-                        when :read_with_stderr then readio
-                        when :stderr then stderrio
-                        else
-                          raise "BUG: invalid mode must be checked before here: '#{m}'"
-                        end
+          io_obj = case m
+                   when :read then readio
+                   when :write then writeio
+                   when :read_with_stderr then readio
+                   when :stderr then stderrio
+                   else
+                     raise "BUG: invalid mode must be checked before here: '#{m}'"
+                   end
+          io_objects << io_obj
         end
 
         m = Mutex.new
@@ -265,28 +305,54 @@ module Fluent
           m.lock # run after plugin thread get pid, thread instance and i/o
           m.unlock
           begin
-            block.call(*io_objects)
+            @_child_process_processes[pid].alive = true
+            block.call(*io_objects) if block_given?
+            writeio.close if writeio
           rescue EOFError => e
             log.debug "Process exit and I/O closed", title: title, pid: pid, command: command, arguments: arguments
           rescue IOError => e
-            if e.message == 'stream closed'
+            if e.message == 'stream closed' || e.message == 'closed stream' # "closed stream" is of ruby 2.1
               log.debug "Process I/O stream closed", title: title, pid: pid, command: command, arguments: arguments
             else
               log.error "Unexpected I/O error for child process", title: title, pid: pid, command: command, arguments: arguments, error: e
             end
+          rescue Errno::EPIPE => e
+            log.debug "Broken pipe, child process unexpectedly exits", title: title, pid: pid, command: command, arguments: arguments
           rescue => e
             log.warn "Unexpected error while processing I/O for child process", title: title, pid: pid, command: command, error: e
           end
-          process_info = @_child_process_mutex.synchronize do
-            process_info = @_child_process_processes[pid]
-            @_child_process_processes.delete(pid)
-            process_info
+
+          if wait_timeout
+            if wait_thread.join(wait_timeout) # Thread#join returns nil when limit expires
+              # wait_thread successfully exits
+              @_child_process_processes[pid].exit_status = wait_thread.value
+            else
+              log.warn "child process timed out", title: title, pid: pid, command: command, arguments: arguments
+              child_process_kill(@_child_process_processes[pid], force: true)
+              @_child_process_processes[pid].exit_status = wait_thread.value
+            end
+          else
+            @_child_process_processes[pid].exit_status = wait_thread.value # with join
+          end
+          process_info = @_child_process_mutex.synchronize{ @_child_process_processes.delete(pid) }
+
+          cb = process_info.on_exit_callback_mutex.synchronize do
+            cback = process_info.on_exit_callback
+            process_info.on_exit_callback = nil
+            cback
+          end
+          if cb
+            cb.call(process_info.exit_status) rescue nil
           end
-          child_process_kill(process_info, force: true) if process_info && process_info.alive && ::Thread.current[:_fluentd_plugin_helper_child_process_running]
         end
         thread[:_fluentd_plugin_helper_child_process_running] = true
         thread[:_fluentd_plugin_helper_child_process_pid] = pid
-        pinfo = ProcessInfo.new(title, thread, pid, readio, readio_in_use, writeio, writeio_in_use, stderrio, stderrio_in_use, wait_thread, true, nil)
+        pinfo = ProcessInfo.new(
+          title, thread, pid,
+          readio, readio_in_use, writeio, writeio_in_use, stderrio, stderrio_in_use,
+          wait_thread, false, nil, nil, on_exit_callback, Mutex.new
+        )
+
         @_child_process_mutex.synchronize do
           @_child_process_processes[pid] = pinfo
         end