fluentd 0.12.40 → 1.6.2

data/lib/fluent/plugin/in_tcp.rb

@@ -14,21 +14,78 @@
 #    limitations under the License.
 #
 
-require 'cool.io'
+require 'fluent/plugin/input'
 
-require 'fluent/plugin/socket_util'
+module Fluent::Plugin
+  class TcpInput < Input
+    Fluent::Plugin.register_input('tcp', self)
 
-module Fluent
-  class TcpInput < SocketUtil::BaseInput
-    Plugin.register_input('tcp', self)
+    helpers :server, :parser, :extract, :compat_parameters
+
+    desc 'Tag of output events.'
+    config_param :tag, :string
+    desc 'The port to listen to.'
+    config_param :port, :integer, default: 5170
+    desc 'The bind address to listen to.'
+    config_param :bind, :string, default: '0.0.0.0'
+
+    desc "The field name of the client's hostname."
+    config_param :source_host_key, :string, default: nil, deprecated: "use source_hostname_key instead."
+    desc "The field name of the client's hostname."
+    config_param :source_hostname_key, :string, default: nil
+    desc "The field name of the client's address."
+    config_param :source_address_key, :string, default: nil
+
+    config_param :blocking_timeout, :time, default: 0.5
 
-    config_set_default :port, 5170
     desc 'The payload is read up to this character.'
     config_param :delimiter, :string, default: "\n" # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
 
-    def listen(callback)
-      log.info "listening tcp socket on #{@bind}:#{@port}"
-      Coolio::TCPServer.new(@bind, @port, SocketUtil::TcpHandler, log, @delimiter, callback, !!@source_hostname_key)
+    def configure(conf)
+      compat_parameters_convert(conf, :parser)
+      parser_config = conf.elements('parse').first
+      unless parser_config
+        raise Fluent::ConfigError, "<parse> section is required."
+      end
+      super
+      @_event_loop_blocking_timeout = @blocking_timeout
+      @source_hostname_key ||= @source_host_key if @source_host_key
+
+      @parser = parser_create(conf: parser_config)
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def start
+      super
+
+      server_create(:in_tcp_server, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|
+        conn.buffer << data
+        begin
+          pos = 0
+          while i = conn.buffer.index(@delimiter, pos)
+            msg = conn.buffer[pos...i]
+            pos = i + @delimiter.length
+
+            @parser.parse(msg) do |time, record|
+              unless time && record
+                log.warn "pattern not matched", message: msg
+                next
+              end
+
+              tag = extract_tag_from_record(record)
+              tag ||= @tag
+              time ||= extract_time_from_record(record) || Fluent::EventTime.now
+              record[@source_address_key] = conn.remote_addr if @source_address_key
+              record[@source_hostname_key] = conn.remote_host if @source_hostname_key
+              router.emit(tag, time, record)
+            end
+          end
+          conn.buffer.slice!(0, pos) if pos > 0
+        end
+      end
     end
   end
 end

data/lib/fluent/plugin/in_udp.rb

@@ -14,30 +14,79 @@
 #    limitations under the License.
 #
 
-require 'fluent/plugin/socket_util'
+require 'fluent/plugin/input'
 
-module Fluent
-  class UdpInput < SocketUtil::BaseInput
-    Plugin.register_input('udp', self)
+module Fluent::Plugin
+  class UdpInput < Input
+    Fluent::Plugin.register_input('udp', self)
 
-    config_set_default :port, 5160
+    helpers :server, :parser, :extract, :compat_parameters
+
+    desc 'Tag of output events.'
+    config_param :tag, :string
+    desc 'The port to listen to.'
+    config_param :port, :integer, default: 5160
+    desc 'The bind address to listen to.'
+    config_param :bind, :string, default: '0.0.0.0'
+
+    desc "The field name of the client's hostname."
+    config_param :source_host_key, :string, default: nil, deprecated: "use source_hostname_key instead."
+    desc "The field name of the client's hostname."
+    config_param :source_hostname_key, :string, default: nil
+    desc "The field name of the client's address."
+    config_param :source_address_key, :string, default: nil
 
     desc "Deprecated parameter. Use message_length_limit instead"
     config_param :body_size_limit, :size, default: nil, deprecated: "use message_length_limit instead."
     desc "The max bytes of message"
     config_param :message_length_limit, :size, default: 4096
+    desc "Remove newline from the end of incoming payload"
+    config_param :remove_newline, :bool, default: true
+    desc "The max size of socket receive buffer. SO_RCVBUF"
+    config_param :receive_buffer_size, :size, default: nil
+
+    config_param :blocking_timeout, :time, default: 0.5
 
     def configure(conf)
+      compat_parameters_convert(conf, :parser)
+      parser_config = conf.elements('parse').first
+      unless parser_config
+        raise Fluent::ConfigError, "<parse> section is required."
+      end
       super
-
+      @_event_loop_blocking_timeout = @blocking_timeout
+      @source_hostname_key ||= @source_host_key if @source_host_key
       @message_length_limit = @body_size_limit if @body_size_limit
+
+      @parser = parser_create(conf: parser_config)
     end
 
-    def listen(callback)
-      log.info "listening udp socket on #{@bind}:#{@port}"
-      @usock = SocketUtil.create_udp_socket(@bind)
-      @usock.bind(@bind, @port)
-      SocketUtil::UdpHandler.new(@usock, log, @message_length_limit, callback, !!@source_hostname_key)
+    def multi_workers_ready?
+      true
+    end
+
+    def start
+      super
+
+      log.info "listening udp socket", bind: @bind, port: @port
+      server_create(:in_udp_server, @port, proto: :udp, bind: @bind, resolve_name: !!@source_hostname_key, max_bytes: @message_length_limit, receive_buffer_size: @receive_buffer_size) do |data, sock|
+        data.chomp! if @remove_newline
+        begin
+          @parser.parse(data) do |time, record|
+            unless time && record
+              log.warn "pattern not matched", data: data
+              next
+            end
+
+            tag = extract_tag_from_record(record)
+            tag ||= @tag
+            time ||= extract_time_from_record(record) || Fluent::EventTime.now
+            record[@source_address_key] = sock.remote_addr if @source_address_key
+            record[@source_hostname_key] = sock.remote_host if @source_hostname_key
+            router.emit(tag, time, record)
+          end
+        end
+      end
     end
   end
 end

data/lib/fluent/plugin/{in_stream.rb → in_unix.rb}

@@ -35,6 +35,8 @@ module Fluent
     end
 
     def start
+      super
+
       @loop = Coolio::Loop.new
       @lsock = listen
       @loop.attach(@lsock)
@@ -46,6 +48,8 @@ module Fluent
       @loop.stop
       @lsock.close
       @thread.join
+
+      super
     end
 
     #def listen
@@ -96,8 +100,8 @@ module Fluent
         entries.each {|e|
           record = e[1]
           next if record.nil?
-          time = e[0].to_i
-          time = (now ||= Engine.now) if time == 0
+          time = e[0]
+          time = (now ||= Engine.now) if time.to_i == 0
           es.add(time, record)
         }
         router.emit_stream(tag, es)
@@ -108,7 +112,7 @@ module Fluent
         return if record.nil?
 
         time = msg[1]
-        time = Engine.now if time == 0
+        time = Engine.now if time.to_i == 0
         router.emit(tag, time, record)
       end
     end
@@ -142,7 +146,7 @@ module Fluent
           @u = Fluent::Engine.msgpack_factory.unpacker
         end
 
-        (class << self; self; end).module_eval do
+        singleton_class.module_eval do
           define_method(:on_read, m)
         end
         m.call(data)

data/lib/fluent/plugin/input.rb

@@ -0,0 +1,37 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+
+require 'fluent/log'
+require 'fluent/plugin_id'
+require 'fluent/plugin_helper'
+
+module Fluent
+  module Plugin
+    class Input < Base
+      include PluginId
+      include PluginLoggerMixin
+      include PluginHelper::Mixin
+
+      helpers_internal :event_emitter
+
+      def multi_workers_ready?
+        false
+      end
+    end
+  end
+end

data/lib/fluent/plugin/multi_output.rb

@@ -0,0 +1,158 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+require 'fluent/log'
+require 'fluent/plugin_id'
+require 'fluent/plugin_helper'
+
+module Fluent
+  module Plugin
+    class MultiOutput < Base
+      include PluginId
+      include PluginLoggerMixin
+      include PluginHelper::Mixin # for event_emitter
+
+      helpers :event_emitter # to get router from agent, which will be supplied to child plugins
+
+      config_section :store, param_name: :stores, multi: true, required: true do
+        config_argument :arg, :string, default: ''
+        config_param :@type, :string, default: nil
+      end
+
+      attr_reader :outputs, :outputs_statically_created
+
+      def process(tag, es)
+        raise NotImplementedError, "BUG: output plugins MUST implement this method"
+      end
+
+      def initialize
+        super
+        @outputs = []
+        @outputs_statically_created = false
+
+        @counters_monitor = Monitor.new
+        # TODO: well organized counters
+        @num_errors = 0
+        @emit_count = 0
+        @emit_records = 0
+        # @write_count = 0
+        # @rollback_count = 0
+      end
+
+      def multi_output?
+        true
+      end
+
+      def configure(conf)
+        super
+
+        @stores.each do |store|
+          store_conf = store.corresponding_config_element
+          type = store_conf['@type']
+          unless type
+            raise Fluent::ConfigError, "Missing '@type' parameter in <store> section"
+          end
+
+          log.debug "adding store", type: type
+
+          output = Fluent::Plugin.new_output(type)
+          output.context_router = self.context_router
+          output.configure(store_conf)
+          @outputs << output
+        end
+      end
+
+      def static_outputs
+        @outputs_statically_created = true
+        @outputs
+      end
+
+      # Child plugin's lifecycles are controlled by agent automatically.
+      # It calls `outputs` to traverse plugins, and invoke start/stop/*shutdown/close/terminate on these directly.
+      # * `start` of this plugin will be called after child plugins
+      # * `stop`, `*shutdown`, `close` and `terminate` of this plugin will be called before child plugins
+
+      # But when MultiOutput plugins are created dynamically (by forest plugin or others), agent cannot find
+      # sub-plugins. So child plugins' lifecycles MUST be controlled by MultiOutput plugin itself.
+      # TODO: this hack will be removed at v2.
+      def call_lifecycle_method(method_name, checker_name)
+        return if @outputs_statically_created
+        @outputs.each do |o|
+          begin
+            log.debug "calling #{method_name} on output plugin dynamically created", type: Fluent::Plugin.lookup_type_from_class(o.class), plugin_id: o.plugin_id
+            o.send(method_name) unless o.send(checker_name)
+          rescue Exception => e
+            log.warn "unexpected error while calling #{method_name} on output plugin dynamically created", plugin: o.class, plugin_id: o.plugin_id, error: e
+            log.warn_backtrace
+          end
+        end
+      end
+
+      def start
+        super
+        call_lifecycle_method(:start, :started?)
+      end
+
+      def after_start
+        super
+        call_lifecycle_method(:after_start, :after_started?)
+      end
+
+      def stop
+        super
+        call_lifecycle_method(:stop, :stopped?)
+      end
+
+      def before_shutdown
+        super
+        call_lifecycle_method(:before_shutdown, :before_shutdown?)
+      end
+
+      def shutdown
+        super
+        call_lifecycle_method(:shutdown, :shutdown?)
+      end
+
+      def after_shutdown
+        super
+        call_lifecycle_method(:after_shutdown, :after_shutdown?)
+      end
+
+      def close
+        super
+        call_lifecycle_method(:close, :closed?)
+      end
+
+      def terminate
+        super
+        call_lifecycle_method(:terminate, :terminated?)
+      end
+
+      def emit_sync(tag, es)
+        @counters_monitor.synchronize{ @emit_count += 1 }
+        begin
+          process(tag, es)
+          @counters_monitor.synchronize{ @emit_records += es.size }
+        rescue
+          @counters_monitor.synchronize{ @num_errors += 1 }
+          raise
+        end
+      end
+      alias :emit_events :emit_sync
+    end
+  end
+end

data/lib/fluent/plugin/out_copy.rb

@@ -14,68 +14,56 @@
 #    limitations under the License.
 #
 
-require 'fluent/output'
+require 'fluent/plugin/multi_output'
 require 'fluent/config/error'
 require 'fluent/event'
 
-module Fluent
+module Fluent::Plugin
   class CopyOutput < MultiOutput
-    Plugin.register_output('copy', self)
+    Fluent::Plugin.register_output('copy', self)
 
     desc 'If true, pass different record to each `store` plugin.'
     config_param :deep_copy, :bool, default: false
 
+    attr_reader :ignore_errors
+
     def initialize
       super
-      @outputs = []
+      @ignore_errors = []
     end
 
-    attr_reader :outputs
-
     def configure(conf)
       super
-      conf.elements.select {|e|
-        e.name == 'store'
-      }.each {|e|
-        type = e['@type'] || e['type']
-        unless type
-          raise ConfigError, "Missing 'type' parameter on <store> directive"
-        end
-        log.debug "adding store type=#{type.dump}"
 
-        output = Plugin.new_output(type)
-        output.router = router
-        output.configure(e)
-        @outputs << output
+      @stores.each { |store|
+        @ignore_errors << (store.arg == 'ignore_error')
       }
     end
 
-    def start
-      @outputs.each {|o|
-        o.start
-      }
+    def multi_workers_ready?
+      true
     end
 
-    def shutdown
-      @outputs.each {|o|
-        o.shutdown
-      }
-    end
-
-    def emit(tag, es, chain)
+    def process(tag, es)
       unless es.repeatable?
-        m = MultiEventStream.new
+        m = Fluent::MultiEventStream.new
         es.each {|time,record|
           m.add(time, record)
         }
         es = m
       end
-      if @deep_copy
-        chain = CopyOutputChain.new(@outputs, tag, es, chain)
-      else
-        chain = OutputChain.new(@outputs, tag, es, chain)
+
+      outputs.each_with_index do |output, i|
+        begin
+          output.emit_events(tag, @deep_copy ? es.dup : es)
+        rescue => e
+          if @ignore_errors[i]
+            log.error "ignore emit error", error: e
+          else
+            raise e
+          end
+        end
       end
-      chain.next
     end
   end
 end

data/lib/fluent/plugin/out_exec.rb

@@ -16,93 +16,90 @@
 
 require 'tempfile'
 
-require 'fluent/output'
+require 'fluent/plugin/output'
 require 'fluent/config/error'
-require 'fluent/plugin/exec_util'
 
-module Fluent
-  class ExecOutput < TimeSlicedOutput
-    Plugin.register_output('exec', self)
+module Fluent::Plugin
+  class ExecOutput < Output
+    Fluent::Plugin.register_output('exec', self)
 
-    def initialize
-      super
-      @localtime = false
-    end
+    helpers :inject, :formatter, :compat_parameters, :child_process
 
-    desc 'The command (program) to execute. The exec plugin passes the path of a TSV file as the last argumen'
+    desc 'The command (program) to execute. The exec plugin passes the path of a TSV file as the last argument'
     config_param :command, :string
-    desc 'Specify the comma-separated keys when using the tsv format.'
-    config_param :keys, default: [] do |val|
-      val.split(',')
+
+    config_param :command_timeout, :time, default: 270 # 4min 30sec
+
+    config_section :format do
+      config_set_default :@type, 'tsv'
     end
-    desc 'The name of the key to use as the event tag. This replaces the value in the event record.'
-    config_param :tag_key, :string, default: nil
-    desc 'The name of the key to use as the event time. This replaces the the value in the event record.'
-    config_param :time_key, :string, default: nil
-    desc 'The format for event time used when the time_key parameter is specified. The default is UNIX time (integer).'
-    config_param :time_format, :string, default: nil
-    desc "The format used to map the incoming events to the program input. (#{ExecUtil::SUPPORTED_FORMAT.keys.join(',')})"
-    config_param :format, default: :tsv do |val|
-      f = ExecUtil::SUPPORTED_FORMAT[val]
-      raise ConfigError, "Unsupported format '#{val}'" unless f
-      f
+
+    config_section :inject do
+      config_set_default :time_type, :string
+      config_set_default :localtime, false
     end
 
-    def configure(conf)
-      super
+    config_section :buffer do
+      config_set_default :delayed_commit_timeout, 300 # 5 min
+    end
 
-      @formatter = case @format
-                   when :tsv
-                     if @keys.empty?
-                       raise ConfigError, "keys option is required on exec output for tsv format"
-                     end
-                     ExecUtil::TSVFormatter.new(@keys)
-                   when :json
-                     ExecUtil::JSONFormatter.new
-                   when :msgpack
-                     ExecUtil::MessagePackFormatter.new
-                   end
+    attr_reader :formatter # for tests
 
-      if @time_key
-        if @time_format
-          tf = TimeFormatter.new(@time_format, @localtime, @timezone)
-          @time_format_proc = tf.method(:format)
-        else
-          @time_format_proc = Proc.new { |time| time.to_s }
-        end
-      end
+    def configure(conf)
+      compat_parameters_convert(conf, :inject, :formatter, :buffer, default_chunk_key: 'time')
+      super
+      @formatter = formatter_create
     end
 
-    def format(tag, time, record)
-      out = ''
-      if @time_key
-        record[@time_key] = @time_format_proc.call(time)
-      end
-      if @tag_key
-        record[@tag_key] = tag
-      end
-      @formatter.call(record, out)
-      out
+    def multi_workers_ready?
+      true
     end
 
-    def write(chunk)
-      if chunk.respond_to?(:path)
-        prog = "#{@command} #{chunk.path}"
+    NEWLINE = "\n"
+
+    def format(tag, time, record)
+      record = inject_values_to_record(tag, time, record)
+      if @formatter.formatter_type == :text_per_line
+        @formatter.format(tag, time, record).chomp + NEWLINE
       else
-        tmpfile = Tempfile.new("fluent-plugin-exec-")
-        tmpfile.binmode
-        chunk.write_to(tmpfile)
-        tmpfile.close
-        prog = "#{@command} #{tmpfile.path}"
+        @formatter.format(tag, time, record)
       end
+    end
 
-      system(prog)
-      ecode = $?.to_i
-      tmpfile.delete if tmpfile
-
-      if ecode != 0
-        raise "command returns #{ecode}: #{prog}"
-      end
+    def try_write(chunk)
+      tmpfile = nil
+      prog = if chunk.respond_to?(:path)
+               "#{@command} #{chunk.path}"
+             else
+               tmpfile = Tempfile.new("fluent-plugin-out-exec-")
+               tmpfile.binmode
+               chunk.write_to(tmpfile)
+               tmpfile.close
+               "#{@command} #{tmpfile.path}"
+             end
+      chunk_id = chunk.unique_id
+      callback = ->(status){
+        begin
+          if tmpfile
+            tmpfile.delete rescue nil
+          end
+          if status && status.success?
+            commit_write(chunk_id)
+          elsif status
+            # #rollback_write will be done automatically if it isn't called at here.
+            # But it's after command_timeout, and this timeout should be longer than users expectation.
+            # So here, this plugin calls it explicitly.
+            rollback_write(chunk_id)
+            log.warn "command exits with error code", prog: prog, status: status.exitstatus, signal: status.termsig
+          else
+            rollback_write(chunk_id)
+            log.warn "command unexpectedly exits without exit status", prog: prog
+          end
+        rescue => e
+          log.error "unexpected error in child process callback", error: e
+        end
+      }
+      child_process_execute(:out_exec_process, prog, stderr: :connect, immediate: true, parallel: true, mode: [], wait_timeout: @command_timeout, on_exit_callback: callback)
     end
   end
 end