fluentd 0.12.43 → 0.14.0

data/lib/fluent/plugin/formatter_ltsv.rb

@@ -0,0 +1,42 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class LabeledTSVFormatter < Formatter
+      Plugin.register_formatter('ltsv', self)
+
+      # http://ltsv.org/
+      include HandleTagAndTimeMixin
+
+      config_param :delimiter, :string, default: "\t"
+      config_param :label_delimiter, :string, default: ":"
+
+      # TODO: escaping for \t in values
+      def format(tag, time, record)
+        filter_record(tag, time, record)
+        formatted = record.inject('') { |result, pair|
+          result << @delimiter if result.length.nonzero?
+          result << "#{pair.first}#{@label_delimiter}#{pair.last}"
+        }
+        formatted << "\n"
+        formatted
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_msgpack.rb

@@ -0,0 +1,32 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class MessagePackFormatter < Formatter
+      Plugin.register_formatter('msgpack', self)
+
+      include HandleTagAndTimeMixin
+      include StructuredFormatMixin
+
+      def format_record(record)
+        record.to_msgpack
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_out_file.rb

@@ -0,0 +1,45 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class OutFileFormatter < Formatter
+      Plugin.register_formatter('out_file', self)
+
+      include HandleTagAndTimeMixin
+
+      config_param :output_time, :bool, default: true
+      config_param :output_tag, :bool, default: true
+      config_param :delimiter, default: "\t" do |val|
+        case val
+        when /SPACE/i then ' '
+        when /COMMA/i then ','
+        else "\t"
+        end
+      end
+
+      def format(tag, time, record)
+        filter_record(tag, time, record)
+        header = ''
+        header << "#{@timef.format(time)}#{@delimiter}" if @output_time
+        header << "#{tag}#{@delimiter}" if @output_tag
+        "#{header}#{Yajl.dump(record)}\n"
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_single_value.rb

@@ -0,0 +1,34 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class SingleValueFormatter < Formatter
+      Plugin.register_formatter('single_value', self)
+
+      config_param :message_key, :string, default: 'message'
+      config_param :add_newline, :bool, default: true
+
+      def format(tag, time, record)
+        text = record[@message_key].to_s.dup
+        text << "\n" if @add_newline
+        text
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_stdout.rb

@@ -0,0 +1,39 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class StdoutFormatter < Formatter
+      Plugin.register_formatter('stdout', self)
+
+      config_param :output_type, :string, default: 'json'
+
+      def configure(conf)
+        super
+
+        @sub_formatter = Plugin.new_formatter(@output_type)
+        @sub_formatter.configure(conf)
+      end
+
+      def format(tag, time, record)
+        header = "#{Time.now.localtime} #{tag}: "
+        "#{header}#{@sub_formatter.format(tag, time, record)}"
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_debug_agent.rb

@@ -42,6 +42,8 @@ module Fluent
     end
 
     def start
+      super
+
       if @unix_path
         require 'drb/unix'
         uri = "drbunix:#{@unix_path}"
@@ -55,6 +57,8 @@ module Fluent
 
     def shutdown
       @server.stop_service if @server
+
+      super
     end
   end
 end

data/lib/fluent/plugin/in_dummy.rb

@@ -16,13 +16,15 @@
 
 require 'json'
 
-require 'fluent/input'
+require 'fluent/plugin/input'
 require 'fluent/config/error'
 
-module Fluent
+module Fluent::Plugin
   class DummyInput < Input
     Fluent::Plugin.register_input('dummy', self)
 
+    helpers :thread, :storage
+
     BIN_NUM = 10
 
     desc "The value is the tag assigned to the generated events."
@@ -35,11 +37,11 @@ module Fluent
     config_param :dummy, default: [{"message"=>"dummy"}] do |val|
       begin
         parsed = JSON.parse(val)
-      rescue JSON::ParserError => e
+      rescue JSON::ParserError => ex
         # Fluent::ConfigParseError, "got incomplete JSON" will be raised
         # at literal_parser.rb with --use-v1-config, but I had to
         # take care at here for the case of --use-v0-config.
-        raise Fluent::ConfigError, "#{e.class}: #{e.message}"
+        raise Fluent::ConfigError, "#{ex.class}: #{ex.message}"
       end
       dummy = parsed.is_a?(Array) ? parsed : [parsed]
       dummy.each_with_index do |e, i|
@@ -48,36 +50,39 @@ module Fluent
       dummy
     end
 
-    def configure(conf)
+    def initialize
       super
+      @storage = nil
+    end
 
-      @increment_value = 0
+    def configure(conf)
+      super
       @dummy_index = 0
     end
 
     def start
       super
-      @running = true
-      @thread = Thread.new(&method(:run))
-    end
 
-    def shutdown
-      @running = false
-      @thread.join
+      @storage = storage_create(type: 'local')
+      if @auto_increment_key && !@storage.get(:auto_increment_value)
+        @storage.put(:auto_increment_value, -1)
+      end
+
+      thread_create(:dummy_input, &method(:run))
     end
 
     def run
       batch_num    = (@rate / BIN_NUM).to_i
       residual_num = (@rate % BIN_NUM)
-      while @running
+      while thread_current_running?
         current_time = Time.now.to_i
         BIN_NUM.times do
-          break unless (@running && Time.now.to_i <= current_time)
+          break unless (thread_current_running? && Time.now.to_i <= current_time)
           wait(0.1) { emit(batch_num) }
         end
-        emit(residual_num)
+        emit(residual_num) if thread_current_running?
         # wait for next second
-        while @running && Time.now.to_i <= current_time
+        while thread_current_running? && Time.now.to_i <= current_time
           sleep 0.01
         end
       end
@@ -96,8 +101,7 @@ module Fluent
       @dummy_index += 1
       if @auto_increment_key
         d = d.dup
-        d[@auto_increment_key] = @increment_value
-        @increment_value += 1
+        d[@auto_increment_key] = @storage.update(:auto_increment_value){|v| v + 1 }
       end
       d
     end

data/lib/fluent/plugin/in_exec.rb

@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'strptime'
 require 'yajl'
 
 require 'fluent/input'
@@ -52,9 +53,9 @@ module Fluent
     def configure(conf)
       super
 
-      if localtime = conf['localtime']
+      if conf['localtime']
         @localtime = true
-      elsif utc = conf['utc']
+      elsif conf['utc']
         @localtime = false
       end
 
@@ -70,9 +71,15 @@ module Fluent
       if @time_key
         if @time_format
           f = @time_format
-          @time_parse_proc = Proc.new {|str| Time.strptime(str, f).to_i }
+          @time_parse_proc =
+            begin
+              strptime = Strptime.new(f)
+              Proc.new { |str| Fluent::EventTime.from_time(strptime.exec(str)) }
+            rescue
+              Proc.new {|str| Fluent::EventTime.from_time(Time.strptime(str, f)) }
+            end
         else
-          @time_parse_proc = Proc.new {|str| str.to_i }
+          @time_parse_proc = Proc.new {|str| Fluent::EventTime.from_time(Time.at(str.to_f)) }
         end
       end
 
@@ -96,6 +103,8 @@ module Fluent
     end
 
     def start
+      super
+
       if @run_interval
         @finished = false
         @thread = Thread.new(&method(:run_periodic))
@@ -127,6 +136,8 @@ module Fluent
         end
         @thread.join
       end
+
+      super
     end
 
     def run
@@ -140,11 +151,10 @@ module Fluent
           io = IO.popen(@command, "r")
           @parser.call(io)
           Process.waitpid(io.pid)
+          sleep @run_interval
         rescue
-          log.error "exec failed to run or shutdown child process", error: $!.to_s, error_class: $!.class.to_s
+          log.error "exec failed to run or shutdown child process", error: $!
           log.warn_backtrace $!.backtrace
-        ensure
-          sleep @run_interval
         end
       end
     end
@@ -170,7 +180,7 @@ module Fluent
 
       router.emit(tag, time, record)
     rescue => e
-      log.error "exec failed to emit", error: e.to_s, error_class: e.class.to_s, tag: tag, record: Yajl.dump(record)
+      log.error "exec failed to emit", error: e, tag: tag, record: Yajl.dump(record)
     end
   end
 end

data/lib/fluent/plugin/in_forward.rb

@@ -25,13 +25,15 @@ module Fluent
   class ForwardInput < Input
     Plugin.register_input('forward', self)
 
+    LISTEN_PORT = 24224
+
     def initialize
       super
       require 'fluent/plugin/socket_util'
     end
 
     desc 'The port to listen to.'
-    config_param :port, :integer, default: DEFAULT_LISTEN_PORT
+    config_param :port, :integer, default: LISTEN_PORT
     desc 'The bind address to listen to.'
     config_param :bind, :string, default: '0.0.0.0'
     config_param :backlog, :integer, default: nil
@@ -47,34 +49,26 @@ module Fluent
     config_param :chunk_size_limit, :size, default: nil
     desc 'Skip an event if incoming event is invalid.'
     config_param :skip_invalid_event, :bool, default: false
-    desc 'Try to resolve hostname from IP addresses or not.'
-    config_param :resolve_hostname, :bool, default: nil
-    desc "The field name of the client's source address."
-    config_param :source_address_key, :string, default: nil
-    desc "The field name of the client's hostname."
-    config_param :source_hostname_key, :string, default: nil
 
     def configure(conf)
       super
-
-      if @source_hostname_key
-        if @resolve_hostname.nil?
-          @resolve_hostname = true
-        elsif !@resolve_hostname # user specifies "false" in configure
-          raise Fluent::ConfigError, "resolve_hostname must be true with source_hostname_key"
-        end
-      end
-      @enable_field_injection = @source_address_key || @source_hostname_key
     end
 
     def start
+      super
+
       @loop = Coolio::Loop.new
 
-      @lsock = listen
+      socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
+      if Fluent.windows?
+        socket_manager_path = socket_manager_path.to_i
+      end
+      client = ServerEngine::SocketManager::Client.new(socket_manager_path)
+
+      @lsock = listen(client)
       @loop.attach(@lsock)
 
-      @usock = SocketUtil.create_udp_socket(@bind)
-      @usock.bind(@bind, @port)
+      @usock = client.listen_udp(@bind, @port)
       @usock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK)
       @hbr = HeartbeatRequestHandler.new(@usock, method(:on_heartbeat_request))
       @loop.attach(@hbr)
@@ -97,11 +91,14 @@ module Fluent
       @usock.close
       @thread.join
       @lsock.close
+
+      super
     end
 
-    def listen
+    def listen(client)
       log.info "listening fluent socket on #{@bind}:#{@port}"
-      s = Coolio::TCPServer.new(@bind, @port, Handler, @linger_timeout, log, @resolve_hostname, method(:on_message))
+      sock = client.listen_tcp(@bind, @port)
+      s = Coolio::TCPServer.new(sock, nil, Handler, @linger_timeout, log, method(:on_message))
       s.listen(@backlog) unless @backlog.nil?
       s
     end
@@ -119,7 +116,7 @@ module Fluent
     def run
       @loop.run(@blocking_timeout)
     rescue => e
-      log.error "unexpected error", error: e, error_class: e.class
+      log.error "unexpected error", error: e
       log.error_backtrace
     end
 
@@ -148,7 +145,7 @@ module Fluent
     #   3: object record
     #   4: object option (optional)
     # }
-    def on_message(msg, chunk_size, peeraddr)
+    def on_message(msg, chunk_size, source)
       if msg.nil?
         # for future TCP heartbeat_request
         return
@@ -156,7 +153,7 @@ module Fluent
 
       # TODO: raise an exception if broken chunk is generated by recoverable situation
       unless msg.is_a?(Array)
-        log.warn "incoming chunk is broken:", source: source_message(peeraddr), msg: msg
+        log.warn "incoming chunk is broken:", source: source, msg: msg
         return
       end
 
@@ -164,24 +161,23 @@ module Fluent
       entries = msg[1]
 
       if @chunk_size_limit && (chunk_size > @chunk_size_limit)
-        log.warn "Input chunk size is larger than 'chunk_size_limit', dropped:", tag: tag, source: source_message(peeraddr), limit: @chunk_size_limit, size: chunk_size
+        log.warn "Input chunk size is larger than 'chunk_size_limit', dropped:", tag: tag, source: source, limit: @chunk_size_limit, size: chunk_size
         return
       elsif @chunk_size_warn_limit && (chunk_size > @chunk_size_warn_limit)
-        log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag, source: source_message(peeraddr), limit: @chunk_size_warn_limit, size: chunk_size
+        log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag, source: source, limit: @chunk_size_warn_limit, size: chunk_size
       end
 
       if entries.class == String
         # PackedForward
         es = MessagePackEventStream.new(entries)
-        es = check_and_skip_invalid_event(tag, es, peeraddr) if @skip_invalid_event
-        es = add_source_host(es, peeraddr) if @enable_field_injection
+        es = check_and_skip_invalid_event(tag, es, source) if @skip_invalid_event
         router.emit_stream(tag, es)
         option = msg[2]
 
       elsif entries.class == Array
         # Forward
         es = if @skip_invalid_event
-               check_and_skip_invalid_event(tag, entries, peeraddr)
+               check_and_skip_invalid_event(tag, entries, source)
              else
                es = MultiEventStream.new
                entries.each { |e|
@@ -193,7 +189,6 @@ module Fluent
                }
                es
              end
-        es = add_source_host(es, peeraddr) if @enable_field_injection
         router.emit_stream(tag, es)
         option = msg[2]
 
@@ -202,15 +197,11 @@ module Fluent
         time = msg[1]
         record = msg[2]
         if @skip_invalid_event && invalid_event?(tag, time, record)
-          log.warn "got invalid event and drop it:", source: source_message(peeraddr), tag: tag, time: time, record: record
+          log.warn "got invalid event and drop it:", source: source, tag: tag, time: time, record: record
           return msg[3] # retry never succeeded so return ack and drop incoming event.
         end
         return if record.nil?
-        time = Engine.now if time == 0
-        if @enable_field_injection
-          record[@source_hostname_key] = peeraddr[2] if @source_hostname_key
-          record[@source_address_key] = peeraddr[3] if @source_address_key
-        end
+        time = Engine.now if time.to_i == 0
         router.emit(tag, time, record)
         option = msg[3]
       end
@@ -220,14 +211,14 @@ module Fluent
     end
 
     def invalid_event?(tag, time, record)
-      !(time.is_a?(Integer) && record.is_a?(Hash) && tag.is_a?(String))
+      !((time.is_a?(Integer) || time.is_a?(::Fluent::EventTime)) && record.is_a?(Hash) && tag.is_a?(String))
     end
 
-    def check_and_skip_invalid_event(tag, es, peeraddr)
+    def check_and_skip_invalid_event(tag, es, source)
       new_es = MultiEventStream.new
       es.each { |time, record|
         if invalid_event?(tag, time, record)
-          log.warn "skip invalid event:", source: source_message(peeraddr), tag: tag, time: time, record: record
+          log.warn "skip invalid event:", source: source, tag: tag, time: time, record: record
           next
         end
         new_es.add(time, record)
@@ -235,50 +226,16 @@ module Fluent
       new_es
     end
 
-    def add_source_host(es, peeraddr)
-      new_es = MultiEventStream.new
-      if @source_address_key && @source_hostname_key
-        address = peeraddr[3]
-        hostname = peeraddr[2]
-        es.each { |time, record|
-          record[@source_address_key] = address
-          record[@source_hostname_key] = hostname
-          new_es.add(time, record)
-        }
-      elsif @source_address_key
-        address = peeraddr[3]
-        es.each { |time, record|
-          record[@source_address_key] = address
-          new_es.add(time, record)
-        }
-      elsif @source_hostname_key
-        hostname = peeraddr[2]
-        es.each { |time, record|
-          record[@source_hostname_key] = hostname
-          new_es.add(time, record)
-        }
-      else
-        raise "BUG: don't call this method in this case"
-      end
-      new_es
-    end
-
-    def source_message(peeraddr)
-      _, port, host, addr = peeraddr
-      "host: #{host}, addr: #{addr}, port: #{port}"
-    end
-
     class Handler < Coolio::Socket
       PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
 
-      def initialize(io, linger_timeout, log, resolve_hostname, on_message)
+      def initialize(io, linger_timeout, log, on_message)
         super(io)
 
-        @peeraddr = nil
         if io.is_a?(TCPSocket) # for unix domain socket support in the future
-          io.do_not_reverse_lookup = !resolve_hostname unless resolve_hostname.nil?
+          _proto, port, host, addr = ( io.peeraddr rescue PEERADDR_FAILED )
+          @source = "host: #{host}, addr: #{addr}, port: #{port}"
 
-          @peeraddr = (io.peeraddr rescue PEERADDR_FAILED)
           opt = [1, linger_timeout].pack('I!I!')  # { int l_onoff; int l_linger; }
           io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
         end
@@ -289,7 +246,7 @@ module Fluent
         @log.trace {
           begin
             remote_port, remote_addr = *Socket.unpack_sockaddr_in(@_io.getpeername)
-          rescue => e
+          rescue
             remote_port = nil
             remote_addr = nil
           end
@@ -307,7 +264,7 @@ module Fluent
           @serializer = :to_json.to_proc
           @y = Yajl::Parser.new
           @y.on_parse_complete = lambda { |obj|
-            option = @on_message.call(obj, @chunk_counter, @peeraddr)
+            option = @on_message.call(obj, @chunk_counter, @source)
             respond option if option
             @chunk_counter = 0
           }
@@ -335,7 +292,7 @@ module Fluent
       def on_read_msgpack(data)
         @chunk_counter += data.bytesize
         @u.feed_each(data) do |obj|
-          option = @on_message.call(obj, @chunk_counter, @peeraddr)
+          option = @on_message.call(obj, @chunk_counter, @source)
           respond option if option
           @chunk_counter = 0
         end